diff --git a/CREDITS.rst b/CREDITS.rst new file mode 100644 index 0000000..1354774 --- /dev/null +++ b/CREDITS.rst @@ -0,0 +1,10 @@ +Credits +======= + +Logo +:::: + +Created by Sarah GHIGLIANO and available on The Noun Project. + +License: https://creativecommons.org/licenses/by/3.0/us/ +Source: https://thenounproject.com/Ghigliano/collection/animals/?i=320941 diff --git a/README.rst b/README.rst index 946e257..9846250 100644 --- a/README.rst +++ b/README.rst @@ -34,20 +34,19 @@ Data-processing for humans. Bonobo is an extract-transform-load framework for python 3.5+ (see comparisons with other data tools). -Bonobo uses plain old python objects (functions, generators and iterators), allows to link them in a directed graph and -execute them using a parallelized strategy, without having to worry about the underlying complexity. +Bonobo uses plain old python objects (functions, generators and iterators), allows them to be linked together in a directed graph, and then executed using a parallelized strategy, without having to worry about the underlying complexity. -Developpers can focus on writing simple and atomic operations, that are by-design easy to unit-test, while the -framework focus on applying them concurrently to rows of data. +Developers can focus on writing simple and atomic operations, that are easy to unit-test by-design, while the focus of the +framework is to apply them concurrently to rows of data. One thing to note: write pure transformations and you'll be safe. -Bonobo is a young rewrite of an old python2.7 tool that ran millions of transformations per day for years on production, -so as though it may not yet be complete or fully stable (please, allow us to reach 1.0), the basics are there. +Bonobo is a young rewrite of an old python2.7 tool that ran millions of transformations per day for years on production. +Although it may not yet be complete or fully stable (please, allow us to reach 1.0), the basics are there. ---- -*Bonobo is under heavy development, we're making the best efforts to keep the core as stable as possible but we also need to move forward. Please allow us to reach 1.0 stability and our sincere apologies for anything we'd break in the process (feel free to complain on issues, so we notice breakages we did not expect)* +*Bonobo is under heavy development, we're doing our best to keep the core as stable as possible while still moving forward. Please allow us to reach 1.0 stability and our sincere apologies for anything we break in the process (feel free to complain on issues, allowing us to correct breakages we did not expect)* ---- diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index 17b98c2..2d13ab4 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -2,6 +2,10 @@ import mimetypes import os import bonobo +from bonobo.commands.util.arguments import parse_variable_argument +from bonobo.util import require +from bonobo.util.iterators import tuplize +from bonobo.util.python import WorkingDirectoryModulesRegistry SHORTCUTS = { 'csv': 'text/csv', @@ -23,7 +27,7 @@ READER = 'reader' WRITER = 'writer' -def resolve_factory(name, filename, factory_type): +def resolve_factory(name, filename, factory_type, options=None): """ Try to resolve which transformation factory to use for this filename. User eventually provided a name, which has priority, otherwise we try to detect it using the mimetype detection on filename. @@ -42,6 +46,11 @@ def resolve_factory(name, filename, factory_type): if _ext in SHORTCUTS: name = SHORTCUTS[_ext] + if options: + options = dict(map(parse_variable_argument, options)) + else: + options = dict() + if not name in REGISTRY: raise RuntimeError( 'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} .'. @@ -49,19 +58,49 @@ def resolve_factory(name, filename, factory_type): ) if factory_type == READER: - return REGISTRY[name][0] + return REGISTRY[name][0], options elif factory_type == WRITER: - return REGISTRY[name][1] + return REGISTRY[name][1], options else: raise ValueError('Invalid factory type.') -def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None): - reader = resolve_factory(reader, input, READER)(input) - writer = resolve_factory(writer, output, WRITER)(output) +@tuplize +def resolve_filters(filters): + registry = WorkingDirectoryModulesRegistry() + for f in filters: + try: + mod, attr = f.split(':', 1) + yield getattr(registry.require(mod), attr) + except ValueError: + yield getattr(bonobo, f) + + +def execute( + input, + output, + reader=None, + reader_option=None, + writer=None, + writer_option=None, + option=None, + filter=None, +): + reader_factory, reader_option = resolve_factory(reader, input, READER, (option or []) + (reader_option or [])) + + if output == '-': + writer_factory, writer_option = bonobo.PrettyPrinter, {} + else: + writer_factory, writer_option = resolve_factory(writer, output, WRITER, (option or []) + (writer_option or [])) + + filters = resolve_filters(filter) graph = bonobo.Graph() - graph.add_chain(reader, writer) + graph.add_chain( + reader_factory(input, **reader_option), + *filters, + writer_factory(output, **writer_option), + ) return bonobo.run( graph, services={ @@ -71,11 +110,44 @@ def execute(input, output, reader=None, reader_options=None, writer=None, writer def register(parser): - parser.add_argument('input') - parser.add_argument('output') - parser.add_argument('--' + READER, '-r') - parser.add_argument('--' + WRITER, '-w') - # parser.add_argument('--reader-option', '-ro', dest='reader_options') - # parser.add_argument('--writer-option', '-wo', dest='writer_options') - # parser.add_argument('--option', '-o', dest='options') + parser.add_argument('input', help='Input filename.') + parser.add_argument('output', help='Output filename.') + parser.add_argument( + '--' + READER, + '-r', + help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.' + ) + parser.add_argument( + '--' + WRITER, + '-w', + help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).' + ) + parser.add_argument( + '--filter', + '-f', + dest='filter', + action='append', + help='Add a filter between input and output', + ) + parser.add_argument( + '--option', + '-O', + dest='option', + action='append', + help='Add a named option to both reader and writer factories (i.e. foo="bar").', + ) + parser.add_argument( + '--' + READER + '-option', + '-' + READER[0].upper(), + dest=READER + '_option', + action='append', + help='Add a named option to the reader factory.', + ) + parser.add_argument( + '--' + WRITER + '-option', + '-' + WRITER[0].upper(), + dest=WRITER + '_option', + action='append', + help='Add a named option to the writer factory.', + ) return execute diff --git a/bonobo/commands/util/__init__.py b/bonobo/commands/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bonobo/commands/util/arguments.py b/bonobo/commands/util/arguments.py new file mode 100644 index 0000000..435c6f5 --- /dev/null +++ b/bonobo/commands/util/arguments.py @@ -0,0 +1,26 @@ +import json + + +def parse_variable_argument(arg): + try: + key, val = arg.split('=', 1) + except ValueError: + return arg, True + + try: + val = json.loads(val) + except json.JSONDecodeError: + pass + + return key, val + + +def test_parse_variable_argument(): + assert parse_variable_argument('foo=bar') == ('foo', 'bar') + assert parse_variable_argument('foo="bar"') == ('foo', 'bar') + assert parse_variable_argument('sep=";"') == ('sep', ';') + assert parse_variable_argument('foo') == ('foo', True) + + +if __name__ == '__main__': + test_parse_var() diff --git a/bonobo/config/options.py b/bonobo/config/options.py index 065cc9d..cad4ca8 100644 --- a/bonobo/config/options.py +++ b/bonobo/config/options.py @@ -53,13 +53,15 @@ class Option: _creation_counter = 0 - def __init__(self, type=None, *, required=True, positional=False, default=None): + def __init__(self, type=None, *, required=True, positional=False, default=None, __doc__=None): self.name = None self.type = type self.required = required if default is None else False self.positional = positional self.default = default + self.__doc__ = __doc__ or self.__doc__ + # This hack is necessary for python3.5 self._creation_counter = Option._creation_counter Option._creation_counter += 1 diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index e23dd05..fa74e40 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -70,7 +70,21 @@ def _count_counter(self, context): context.send(Bag(counter._value)) +def _shorten(s, w): + if w and len(s) > w: + s = s[0:w - 3] + '...' + return s + + class PrettyPrinter(Configurable): + max_width = Option( + int, + required=False, + __doc__=''' + If set, truncates the output values longer than this to this width. + ''' + ) + def call(self, *args, **kwargs): formater = self._format_quiet if settings.QUIET.get() else self._format_console @@ -82,7 +96,10 @@ class PrettyPrinter(Configurable): def _format_console(self, i, item, value): return ' '.join( - ((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL) + ( + (' ' if i else '•'), str(item), '=', _shorten(str(value).strip(), + self.max_width).replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL + ) ) diff --git a/bonobo/nodes/io/base.py b/bonobo/nodes/io/base.py index 496a0e8..3cecb70 100644 --- a/bonobo/nodes/io/base.py +++ b/bonobo/nodes/io/base.py @@ -50,6 +50,7 @@ class FileHandler(Configurable): eol = Option(str, default='\n') # type: str mode = Option(str) # type: str encoding = Option(str, default='utf-8') # type: str + fs = Service('fs') # type: str @ContextProcessor diff --git a/bonobo/util/python.py b/bonobo/util/python.py index a496e19..8648f16 100644 --- a/bonobo/util/python.py +++ b/bonobo/util/python.py @@ -9,14 +9,23 @@ class _RequiredModule: class _RequiredModulesRegistry(dict): + @property + def pathname(self): + return os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.stack()[2][0]))) + def require(self, name): if name not in self: bits = name.split('.') - pathname = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.stack()[1][0]))) - filename = os.path.join(pathname, *bits[:-1], bits[-1] + '.py') + filename = os.path.join(self.pathname, *bits[:-1], bits[-1] + '.py') self[name] = _RequiredModule(runpy.run_path(filename, run_name=name)) return self[name] +class WorkingDirectoryModulesRegistry(_RequiredModulesRegistry): + @property + def pathname(self): + return os.getcwd() + + registry = _RequiredModulesRegistry() require = registry.require diff --git a/docs/_templates/alabaster/__init__.py b/docs/_templates/alabaster/__init__.py index 39f1407..a4bebf2 100644 --- a/docs/_templates/alabaster/__init__.py +++ b/docs/_templates/alabaster/__init__.py @@ -14,11 +14,11 @@ def get_path(): def update_context(app, pagename, templatename, context, doctree): context['alabaster_version'] = version.__version__ + def setup(app): # add_html_theme is new in Sphinx 1.6+ if hasattr(app, 'add_html_theme'): theme_path = os.path.abspath(os.path.dirname(__file__)) app.add_html_theme('alabaster', theme_path) app.connect('html-page-context', update_context) - return {'version': version.__version__, - 'parallel_read_safe': True} + return {'version': version.__version__, 'parallel_read_safe': True} diff --git a/docs/_templates/alabaster/support.py b/docs/_templates/alabaster/support.py index 0f3aa8c..9147b52 100644 --- a/docs/_templates/alabaster/support.py +++ b/docs/_templates/alabaster/support.py @@ -7,82 +7,74 @@ from pygments.token import Keyword, Name, Comment, String, Error, \ # Originally based on FlaskyStyle which was based on 'tango'. class Alabaster(Style): - background_color = "#f8f8f8" # doesn't seem to override CSS 'pre' styling? + background_color = "#f8f8f8" # doesn't seem to override CSS 'pre' styling? default_style = "" styles = { # No corresponding class for the following: #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' + Whitespace: "underline #f8f8f8", # class: 'w' + Error: "#a40000 border:#ef2929", # class: 'err' + Other: "#000000", # class 'x' + Comment: "italic #8f5902", # class: 'c' + Comment.Preproc: "noitalic", # class: 'cp' + Keyword: "bold #004461", # class: 'k' + Keyword.Constant: "bold #004461", # class: 'kc' + Keyword.Declaration: "bold #004461", # class: 'kd' + Keyword.Namespace: "bold #004461", # class: 'kn' + Keyword.Pseudo: "bold #004461", # class: 'kp' + Keyword.Reserved: "bold #004461", # class: 'kr' + Keyword.Type: "bold #004461", # class: 'kt' + Operator: "#582800", # class: 'o' + Operator.Word: "bold #004461", # class: 'ow' - like keywords + Punctuation: "bold #000000", # class: 'p' # because special names such as Name.Class, Name.Function, etc. # are not recognized as such later in the parsing, we choose them # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' + Name: "#000000", # class: 'n' + Name.Attribute: "#c4a000", # class: 'na' - to be revised + Name.Builtin: "#004461", # class: 'nb' + Name.Builtin.Pseudo: "#3465a4", # class: 'bp' + Name.Class: "#000000", # class: 'nc' - to be revised + Name.Constant: "#000000", # class: 'no' - to be revised + Name.Decorator: "#888", # class: 'nd' - to be revised + Name.Entity: "#ce5c00", # class: 'ni' + Name.Exception: "bold #cc0000", # class: 'ne' + Name.Function: "#000000", # class: 'nf' + Name.Property: "#000000", # class: 'py' + Name.Label: "#f57900", # class: 'nl' + Name.Namespace: "#000000", # class: 'nn' - to be revised + Name.Other: "#000000", # class: 'nx' + Name.Tag: "bold #004461", # class: 'nt' - like a keyword + Name.Variable: "#000000", # class: 'nv' - to be revised + Name.Variable.Class: "#000000", # class: 'vc' - to be revised + Name.Variable.Global: "#000000", # class: 'vg' - to be revised + Name.Variable.Instance: "#000000", # class: 'vi' - to be revised + Number: "#990000", # class: 'm' + Literal: "#000000", # class: 'l' + Literal.Date: "#000000", # class: 'ld' + String: "#4e9a06", # class: 's' + String.Backtick: "#4e9a06", # class: 'sb' + String.Char: "#4e9a06", # class: 'sc' + String.Doc: "italic #8f5902", # class: 'sd' - like a comment + String.Double: "#4e9a06", # class: 's2' + String.Escape: "#4e9a06", # class: 'se' + String.Heredoc: "#4e9a06", # class: 'sh' + String.Interpol: "#4e9a06", # class: 'si' + String.Other: "#4e9a06", # class: 'sx' + String.Regex: "#4e9a06", # class: 'sr' + String.Single: "#4e9a06", # class: 's1' + String.Symbol: "#4e9a06", # class: 'ss' + Generic: "#000000", # class: 'g' + Generic.Deleted: "#a40000", # class: 'gd' + Generic.Emph: "italic #000000", # class: 'ge' + Generic.Error: "#ef2929", # class: 'gr' + Generic.Heading: "bold #000080", # class: 'gh' + Generic.Inserted: "#00A000", # class: 'gi' + Generic.Output: "#888", # class: 'go' + Generic.Prompt: "#745334", # class: 'gp' + Generic.Strong: "bold #000000", # class: 'gs' + Generic.Subheading: "bold #800080", # class: 'gu' + Generic.Traceback: "bold #a40000", # class: 'gt' } diff --git a/docs/_templates/base.html b/docs/_templates/base.html index 2542eb0..f8ad58a 100644 --- a/docs/_templates/base.html +++ b/docs/_templates/base.html @@ -21,26 +21,19 @@ {{ relbar() }}
- {% if show_copyright %}©{{ copyright }}.{% endif %} - {% if theme_show_powered_by|lower == 'true' %} - {% if show_copyright %}|{% endif %} - Powered by Sphinx {{ sphinx_version }} - & Alabaster {{ alabaster_version }} - {% endif %} - {%- if show_source and has_source and sourcename %} - {% if show_copyright or theme_show_powered_by %}|{% endif %} - {{ _('Page source') }} + © 2012-2017, Romain Dorgueil | + Bonobo ETL + + {%- if show_source and has_source and sourcename %} + | {{ _('Page source') }} {%- endif %}
-{% if theme_github_banner|lower != 'false' %} - + Fork me on GitHub -{% endif %} {% if theme_analytics_id %} {% endif %} + + {%- endblock %} diff --git a/docs/faq.rst b/docs/faq.rst index 5b25c7b..1f24831 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -13,8 +13,7 @@ as input. By default, it uses a thread pool to execute all functions in parallel, and handle the movement of data rows in the directed graph using simple fifo queues. -It allows the user to focus on the content of the transformations, and not optimizing blocking or long operations, nor -thinking about threads or subprocesses. +It allows the user to focus on the content of the transformations, rather than worrying about optimized blocking, long operations, threads, or subprocesses. It's lean manufacturing for data. @@ -34,7 +33,7 @@ The main reasons about why 3.5+: * Creating a tool that works well under both python 2 and 3 is a lot more work. * Python 3 is nearly 10 years old. Consider moving on. -* Python 3.5 contains syntaxic sugar that makes working with data a lot more convenient. +* Python 3.5+ contains syntactic sugar that makes working with data a lot more convenient (and fun). Can a graph contain another graph? diff --git a/docs/guide/introduction.rst b/docs/guide/introduction.rst index 7d89253..dcf9686 100644 --- a/docs/guide/introduction.rst +++ b/docs/guide/introduction.rst @@ -68,6 +68,8 @@ processing while `B` and `C` are working. BEGIN2 -> "B" -> "C"; } +Now, we feed `C` with both `A` and `B` output. It is not a "join", or "cartesian product". It is just two different +pipes plugged to `C` input, and whichever yields data will see this data feeded to `C`, one row at a time. What is it not? :::::::::::::::