From 3b20a40bb77986428b903137c6562b3e6a92aa3f Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 8 Oct 2017 17:42:48 +0200 Subject: [PATCH 01/14] [doc] Missing description. --- docs/guide/introduction.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guide/introduction.rst b/docs/guide/introduction.rst index 7d89253..dcf9686 100644 --- a/docs/guide/introduction.rst +++ b/docs/guide/introduction.rst @@ -68,6 +68,8 @@ processing while `B` and `C` are working. BEGIN2 -> "B" -> "C"; } +Now, we feed `C` with both `A` and `B` output. It is not a "join", or "cartesian product". It is just two different +pipes plugged to `C` input, and whichever yields data will see this data feeded to `C`, one row at a time. What is it not? ::::::::::::::: From 59035fd1813f93aed17b14bb5fc027dbe8b71c09 Mon Sep 17 00:00:00 2001 From: CW Andrews Date: Sun, 8 Oct 2017 12:47:54 -0400 Subject: [PATCH 02/14] Update README.rst English grammar, readability, and spelling corrections (proposed). --- README.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 946e257..9846250 100644 --- a/README.rst +++ b/README.rst @@ -34,20 +34,19 @@ Data-processing for humans. Bonobo is an extract-transform-load framework for python 3.5+ (see comparisons with other data tools). -Bonobo uses plain old python objects (functions, generators and iterators), allows to link them in a directed graph and -execute them using a parallelized strategy, without having to worry about the underlying complexity. +Bonobo uses plain old python objects (functions, generators and iterators), allows them to be linked together in a directed graph, and then executed using a parallelized strategy, without having to worry about the underlying complexity. -Developpers can focus on writing simple and atomic operations, that are by-design easy to unit-test, while the -framework focus on applying them concurrently to rows of data. +Developers can focus on writing simple and atomic operations, that are easy to unit-test by-design, while the focus of the +framework is to apply them concurrently to rows of data. One thing to note: write pure transformations and you'll be safe. -Bonobo is a young rewrite of an old python2.7 tool that ran millions of transformations per day for years on production, -so as though it may not yet be complete or fully stable (please, allow us to reach 1.0), the basics are there. +Bonobo is a young rewrite of an old python2.7 tool that ran millions of transformations per day for years on production. +Although it may not yet be complete or fully stable (please, allow us to reach 1.0), the basics are there. ---- -*Bonobo is under heavy development, we're making the best efforts to keep the core as stable as possible but we also need to move forward. Please allow us to reach 1.0 stability and our sincere apologies for anything we'd break in the process (feel free to complain on issues, so we notice breakages we did not expect)* +*Bonobo is under heavy development, we're doing our best to keep the core as stable as possible while still moving forward. Please allow us to reach 1.0 stability and our sincere apologies for anything we break in the process (feel free to complain on issues, allowing us to correct breakages we did not expect)* ---- From 8bef85704f64237c18a201d283c3311f0e944ece Mon Sep 17 00:00:00 2001 From: CW Andrews Date: Sun, 8 Oct 2017 13:54:31 -0400 Subject: [PATCH 03/14] Update faq.rst Spelling, grammar, and wording changes. --- docs/faq.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 5b25c7b..1f24831 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -13,8 +13,7 @@ as input. By default, it uses a thread pool to execute all functions in parallel, and handle the movement of data rows in the directed graph using simple fifo queues. -It allows the user to focus on the content of the transformations, and not optimizing blocking or long operations, nor -thinking about threads or subprocesses. +It allows the user to focus on the content of the transformations, rather than worrying about optimized blocking, long operations, threads, or subprocesses. It's lean manufacturing for data. @@ -34,7 +33,7 @@ The main reasons about why 3.5+: * Creating a tool that works well under both python 2 and 3 is a lot more work. * Python 3 is nearly 10 years old. Consider moving on. -* Python 3.5 contains syntaxic sugar that makes working with data a lot more convenient. +* Python 3.5+ contains syntactic sugar that makes working with data a lot more convenient (and fun). Can a graph contain another graph? From 0c310543e4697e4332b2278c2908e61b678a99b9 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Tue, 10 Oct 2017 08:30:03 +0200 Subject: [PATCH 04/14] Update base.html --- docs/_templates/base.html | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/docs/_templates/base.html b/docs/_templates/base.html index 2542eb0..fb5d712 100644 --- a/docs/_templates/base.html +++ b/docs/_templates/base.html @@ -21,26 +21,16 @@ {{ relbar() }}
- {% if show_copyright %}©{{ copyright }}.{% endif %} - {% if theme_show_powered_by|lower == 'true' %} - {% if show_copyright %}|{% endif %} - Powered by Sphinx {{ sphinx_version }} - & Alabaster {{ alabaster_version }} - {% endif %} - {%- if show_source and has_source and sourcename %} - {% if show_copyright or theme_show_powered_by %}|{% endif %} - {{ _('Page source') }} - {%- endif %} + © 2012-2017, Romain Dorgueil | + Bonobo ETL | + {{ _('Page source') }}
-{% if theme_github_banner|lower != 'false' %} - + Fork me on GitHub -{% endif %} {% if theme_analytics_id %} {% endif %} + + {%- endblock %} From 2898902ebde81500c5dccc6d2cda259b97312413 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 12 Oct 2017 19:01:35 +0200 Subject: [PATCH 09/14] [cli] adds ability to override reader/writer options from cli convert. --- bonobo/commands/convert.py | 64 ++++++++++++++++++++++++------- bonobo/commands/util/__init__.py | 0 bonobo/commands/util/arguments.py | 26 +++++++++++++ bonobo/nodes/basics.py | 15 +++++++- bonobo/nodes/io/base.py | 1 + 5 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 bonobo/commands/util/__init__.py create mode 100644 bonobo/commands/util/arguments.py diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index 17b98c2..f02e8ab 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -1,7 +1,10 @@ import mimetypes import os +import re + import bonobo +from bonobo.commands.util.arguments import parse_variable_argument SHORTCUTS = { 'csv': 'text/csv', @@ -23,7 +26,7 @@ READER = 'reader' WRITER = 'writer' -def resolve_factory(name, filename, factory_type): +def resolve_factory(name, filename, factory_type, options=None): """ Try to resolve which transformation factory to use for this filename. User eventually provided a name, which has priority, otherwise we try to detect it using the mimetype detection on filename. @@ -42,6 +45,12 @@ def resolve_factory(name, filename, factory_type): if _ext in SHORTCUTS: name = SHORTCUTS[_ext] + if options: + options = dict(map(parse_variable_argument, options)) + + else: + options = dict() + if not name in REGISTRY: raise RuntimeError( 'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} .'. @@ -49,19 +58,22 @@ def resolve_factory(name, filename, factory_type): ) if factory_type == READER: - return REGISTRY[name][0] + return REGISTRY[name][0], options elif factory_type == WRITER: - return REGISTRY[name][1] + return REGISTRY[name][1], options else: raise ValueError('Invalid factory type.') def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None): - reader = resolve_factory(reader, input, READER)(input) - writer = resolve_factory(writer, output, WRITER)(output) + reader_factory, reader_options = resolve_factory(reader, input, READER, (options or []) + (reader_options or [])) + writer_factory, writer_options = resolve_factory(writer, output, WRITER, (options or []) + (writer_options or [])) graph = bonobo.Graph() - graph.add_chain(reader, writer) + graph.add_chain( + reader_factory(input, **reader_options), + writer_factory(output, **writer_options), + ) return bonobo.run( graph, services={ @@ -71,11 +83,37 @@ def execute(input, output, reader=None, reader_options=None, writer=None, writer def register(parser): - parser.add_argument('input') - parser.add_argument('output') - parser.add_argument('--' + READER, '-r') - parser.add_argument('--' + WRITER, '-w') - # parser.add_argument('--reader-option', '-ro', dest='reader_options') - # parser.add_argument('--writer-option', '-wo', dest='writer_options') - # parser.add_argument('--option', '-o', dest='options') + parser.add_argument('input', help='Input filename.') + parser.add_argument('output', help='Output filename.') + parser.add_argument( + '--' + READER, + '-r', + help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.' + ) + parser.add_argument( + '--' + WRITER, + '-w', + help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong.' + ) + parser.add_argument( + '--option', + '-O', + dest='options', + action='append', + help='Add a named option to both reader and writer factories (i.e. foo="bar").' + ) + parser.add_argument( + '--' + READER + '-option', + '-' + READER[0].upper(), + dest=READER + '_options', + action='append', + help='Add a named option to the reader factory.' + ) + parser.add_argument( + '--' + WRITER + '-option', + '-' + WRITER[0].upper(), + dest=WRITER + '_options', + action='append', + help='Add a named option to the writer factory.' + ) return execute diff --git a/bonobo/commands/util/__init__.py b/bonobo/commands/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bonobo/commands/util/arguments.py b/bonobo/commands/util/arguments.py new file mode 100644 index 0000000..435c6f5 --- /dev/null +++ b/bonobo/commands/util/arguments.py @@ -0,0 +1,26 @@ +import json + + +def parse_variable_argument(arg): + try: + key, val = arg.split('=', 1) + except ValueError: + return arg, True + + try: + val = json.loads(val) + except json.JSONDecodeError: + pass + + return key, val + + +def test_parse_variable_argument(): + assert parse_variable_argument('foo=bar') == ('foo', 'bar') + assert parse_variable_argument('foo="bar"') == ('foo', 'bar') + assert parse_variable_argument('sep=";"') == ('sep', ';') + assert parse_variable_argument('foo') == ('foo', True) + + +if __name__ == '__main__': + test_parse_var() diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index e23dd05..b346404 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -70,7 +70,17 @@ def _count_counter(self, context): context.send(Bag(counter._value)) +def _shorten(s, w): + if w and len(s) > w: + s = s[0:w - 3] + '...' + return s + + class PrettyPrinter(Configurable): + max_width = Option(int, required=False, __doc__=''' + If set, truncates the output values longer than this to this width. + ''') + def call(self, *args, **kwargs): formater = self._format_quiet if settings.QUIET.get() else self._format_console @@ -82,7 +92,10 @@ class PrettyPrinter(Configurable): def _format_console(self, i, item, value): return ' '.join( - ((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL) + ( + (' ' if i else '•'), str(item), '=', _shorten(str(value).strip(), + self.max_width).replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL + ) ) diff --git a/bonobo/nodes/io/base.py b/bonobo/nodes/io/base.py index 496a0e8..3cecb70 100644 --- a/bonobo/nodes/io/base.py +++ b/bonobo/nodes/io/base.py @@ -50,6 +50,7 @@ class FileHandler(Configurable): eol = Option(str, default='\n') # type: str mode = Option(str) # type: str encoding = Option(str, default='utf-8') # type: str + fs = Service('fs') # type: str @ContextProcessor From 99351a638bf664c4719569bbdab567496d6d431d Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 12 Oct 2017 19:01:47 +0200 Subject: [PATCH 10/14] [doc] formating --- docs/_templates/alabaster/__init__.py | 4 +- docs/_templates/alabaster/support.py | 130 ++++++++++++-------------- 2 files changed, 63 insertions(+), 71 deletions(-) diff --git a/docs/_templates/alabaster/__init__.py b/docs/_templates/alabaster/__init__.py index 39f1407..a4bebf2 100644 --- a/docs/_templates/alabaster/__init__.py +++ b/docs/_templates/alabaster/__init__.py @@ -14,11 +14,11 @@ def get_path(): def update_context(app, pagename, templatename, context, doctree): context['alabaster_version'] = version.__version__ + def setup(app): # add_html_theme is new in Sphinx 1.6+ if hasattr(app, 'add_html_theme'): theme_path = os.path.abspath(os.path.dirname(__file__)) app.add_html_theme('alabaster', theme_path) app.connect('html-page-context', update_context) - return {'version': version.__version__, - 'parallel_read_safe': True} + return {'version': version.__version__, 'parallel_read_safe': True} diff --git a/docs/_templates/alabaster/support.py b/docs/_templates/alabaster/support.py index 0f3aa8c..9147b52 100644 --- a/docs/_templates/alabaster/support.py +++ b/docs/_templates/alabaster/support.py @@ -7,82 +7,74 @@ from pygments.token import Keyword, Name, Comment, String, Error, \ # Originally based on FlaskyStyle which was based on 'tango'. class Alabaster(Style): - background_color = "#f8f8f8" # doesn't seem to override CSS 'pre' styling? + background_color = "#f8f8f8" # doesn't seem to override CSS 'pre' styling? default_style = "" styles = { # No corresponding class for the following: #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' + Whitespace: "underline #f8f8f8", # class: 'w' + Error: "#a40000 border:#ef2929", # class: 'err' + Other: "#000000", # class 'x' + Comment: "italic #8f5902", # class: 'c' + Comment.Preproc: "noitalic", # class: 'cp' + Keyword: "bold #004461", # class: 'k' + Keyword.Constant: "bold #004461", # class: 'kc' + Keyword.Declaration: "bold #004461", # class: 'kd' + Keyword.Namespace: "bold #004461", # class: 'kn' + Keyword.Pseudo: "bold #004461", # class: 'kp' + Keyword.Reserved: "bold #004461", # class: 'kr' + Keyword.Type: "bold #004461", # class: 'kt' + Operator: "#582800", # class: 'o' + Operator.Word: "bold #004461", # class: 'ow' - like keywords + Punctuation: "bold #000000", # class: 'p' # because special names such as Name.Class, Name.Function, etc. # are not recognized as such later in the parsing, we choose them # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' + Name: "#000000", # class: 'n' + Name.Attribute: "#c4a000", # class: 'na' - to be revised + Name.Builtin: "#004461", # class: 'nb' + Name.Builtin.Pseudo: "#3465a4", # class: 'bp' + Name.Class: "#000000", # class: 'nc' - to be revised + Name.Constant: "#000000", # class: 'no' - to be revised + Name.Decorator: "#888", # class: 'nd' - to be revised + Name.Entity: "#ce5c00", # class: 'ni' + Name.Exception: "bold #cc0000", # class: 'ne' + Name.Function: "#000000", # class: 'nf' + Name.Property: "#000000", # class: 'py' + Name.Label: "#f57900", # class: 'nl' + Name.Namespace: "#000000", # class: 'nn' - to be revised + Name.Other: "#000000", # class: 'nx' + Name.Tag: "bold #004461", # class: 'nt' - like a keyword + Name.Variable: "#000000", # class: 'nv' - to be revised + Name.Variable.Class: "#000000", # class: 'vc' - to be revised + Name.Variable.Global: "#000000", # class: 'vg' - to be revised + Name.Variable.Instance: "#000000", # class: 'vi' - to be revised + Number: "#990000", # class: 'm' + Literal: "#000000", # class: 'l' + Literal.Date: "#000000", # class: 'ld' + String: "#4e9a06", # class: 's' + String.Backtick: "#4e9a06", # class: 'sb' + String.Char: "#4e9a06", # class: 'sc' + String.Doc: "italic #8f5902", # class: 'sd' - like a comment + String.Double: "#4e9a06", # class: 's2' + String.Escape: "#4e9a06", # class: 'se' + String.Heredoc: "#4e9a06", # class: 'sh' + String.Interpol: "#4e9a06", # class: 'si' + String.Other: "#4e9a06", # class: 'sx' + String.Regex: "#4e9a06", # class: 'sr' + String.Single: "#4e9a06", # class: 's1' + String.Symbol: "#4e9a06", # class: 'ss' + Generic: "#000000", # class: 'g' + Generic.Deleted: "#a40000", # class: 'gd' + Generic.Emph: "italic #000000", # class: 'ge' + Generic.Error: "#ef2929", # class: 'gr' + Generic.Heading: "bold #000080", # class: 'gh' + Generic.Inserted: "#00A000", # class: 'gi' + Generic.Output: "#888", # class: 'go' + Generic.Prompt: "#745334", # class: 'gp' + Generic.Strong: "bold #000000", # class: 'gs' + Generic.Subheading: "bold #800080", # class: 'gu' + Generic.Traceback: "bold #a40000", # class: 'gt' } From 721ed499bbe028c8984925fb075845feef445447 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 12 Oct 2017 19:02:11 +0200 Subject: [PATCH 11/14] [config] adds a __doc__ constructor kwarg to set option documentation inline. --- bonobo/config/options.py | 4 +++- bonobo/nodes/basics.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bonobo/config/options.py b/bonobo/config/options.py index 065cc9d..cad4ca8 100644 --- a/bonobo/config/options.py +++ b/bonobo/config/options.py @@ -53,13 +53,15 @@ class Option: _creation_counter = 0 - def __init__(self, type=None, *, required=True, positional=False, default=None): + def __init__(self, type=None, *, required=True, positional=False, default=None, __doc__=None): self.name = None self.type = type self.required = required if default is None else False self.positional = positional self.default = default + self.__doc__ = __doc__ or self.__doc__ + # This hack is necessary for python3.5 self._creation_counter = Option._creation_counter Option._creation_counter += 1 diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index b346404..fa74e40 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -77,9 +77,13 @@ def _shorten(s, w): class PrettyPrinter(Configurable): - max_width = Option(int, required=False, __doc__=''' + max_width = Option( + int, + required=False, + __doc__=''' If set, truncates the output values longer than this to this width. - ''') + ''' + ) def call(self, *args, **kwargs): formater = self._format_quiet if settings.QUIET.get() else self._format_console From b1d7498054da969b81215ab851c7fb25a5575f87 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 12 Oct 2017 19:12:10 +0200 Subject: [PATCH 12/14] [cli] convert, remove useless import. --- bonobo/commands/convert.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index f02e8ab..05d3504 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -1,8 +1,6 @@ import mimetypes import os -import re - import bonobo from bonobo.commands.util.arguments import parse_variable_argument @@ -47,14 +45,13 @@ def resolve_factory(name, filename, factory_type, options=None): if options: options = dict(map(parse_variable_argument, options)) - else: options = dict() if not name in REGISTRY: raise RuntimeError( 'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} .'. - format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]) + format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]) ) if factory_type == READER: From 64875a05bd8cf5bcf64c443269d5dd35d8ee07a1 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Fri, 13 Oct 2017 17:21:25 +0200 Subject: [PATCH 13/14] [cli] Adds a --filter option to "convert" command, allowing to use arbitrary filters to a command line conversion. Also adds --print and "-" output to pretty print to terminal instead of file output. --- bonobo/commands/convert.py | 69 +++++++++++++++++++++++++++++++------- bonobo/util/python.py | 13 +++++-- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index 05d3504..01441e4 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -3,6 +3,9 @@ import os import bonobo from bonobo.commands.util.arguments import parse_variable_argument +from bonobo.util import require +from bonobo.util.iterators import tuplize +from bonobo.util.python import WorkingDirectoryModulesRegistry SHORTCUTS = { 'csv': 'text/csv', @@ -51,7 +54,7 @@ def resolve_factory(name, filename, factory_type, options=None): if not name in REGISTRY: raise RuntimeError( 'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} .'. - format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]) + format(name=name, filename=filename, factory_type=factory_type, opt=factory_type[0]) ) if factory_type == READER: @@ -62,14 +65,42 @@ def resolve_factory(name, filename, factory_type, options=None): raise ValueError('Invalid factory type.') -def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None): - reader_factory, reader_options = resolve_factory(reader, input, READER, (options or []) + (reader_options or [])) - writer_factory, writer_options = resolve_factory(writer, output, WRITER, (options or []) + (writer_options or [])) +@tuplize +def resolve_filters(filters): + registry = WorkingDirectoryModulesRegistry() + for f in filters: + try: + mod, attr = f.split(':', 1) + yield getattr(registry.require(mod), attr) + except ValueError: + yield getattr(bonobo, f) + + +def execute( + input, + output, + reader=None, + reader_option=None, + writer=None, + writer_option=None, + option=None, + filter=None, + do_print=False +): + reader_factory, reader_option = resolve_factory(reader, input, READER, (option or []) + (reader_option or [])) + + if output == '-': + writer_factory, writer_option = bonobo.PrettyPrinter, {} + else: + writer_factory, writer_option = resolve_factory(writer, output, WRITER, (option or []) + (writer_option or [])) + + filters = resolve_filters(filter) graph = bonobo.Graph() graph.add_chain( - reader_factory(input, **reader_options), - writer_factory(output, **writer_options), + reader_factory(input, **reader_option), + *filters, + writer_factory(output, **writer_option), ) return bonobo.run( @@ -92,25 +123,39 @@ def register(parser): '-w', help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong.' ) + parser.add_argument( + '--filter', + '-f', + dest='filter', + action='append', + help='Add a filter between input and output', + ) + parser.add_argument( + '--print', + '-p', + dest='do_print', + action='store_true', + help='Replace the output by a pretty printer.', + ) parser.add_argument( '--option', '-O', - dest='options', + dest='option', action='append', - help='Add a named option to both reader and writer factories (i.e. foo="bar").' + help='Add a named option to both reader and writer factories (i.e. foo="bar").', ) parser.add_argument( '--' + READER + '-option', '-' + READER[0].upper(), - dest=READER + '_options', + dest=READER + '_option', action='append', - help='Add a named option to the reader factory.' + help='Add a named option to the reader factory.', ) parser.add_argument( '--' + WRITER + '-option', '-' + WRITER[0].upper(), - dest=WRITER + '_options', + dest=WRITER + '_option', action='append', - help='Add a named option to the writer factory.' + help='Add a named option to the writer factory.', ) return execute diff --git a/bonobo/util/python.py b/bonobo/util/python.py index a496e19..e55c24c 100644 --- a/bonobo/util/python.py +++ b/bonobo/util/python.py @@ -9,14 +9,23 @@ class _RequiredModule: class _RequiredModulesRegistry(dict): + @property + def pathname(self): + return os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.stack()[1][0]))) + def require(self, name): if name not in self: bits = name.split('.') - pathname = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.stack()[1][0]))) - filename = os.path.join(pathname, *bits[:-1], bits[-1] + '.py') + filename = os.path.join(self.pathname, *bits[:-1], bits[-1] + '.py') self[name] = _RequiredModule(runpy.run_path(filename, run_name=name)) return self[name] +class WorkingDirectoryModulesRegistry(_RequiredModulesRegistry): + @property + def pathname(self): + return os.getcwd() + + registry = _RequiredModulesRegistry() require = registry.require From dc59c88c3d52d51ee8f710a3d1c14ecbaff4e51e Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Fri, 13 Oct 2017 17:25:42 +0200 Subject: [PATCH 14/14] [cli/util] fix requires to use the right stack frame, remove --print as "-" does the job --- bonobo/commands/convert.py | 10 +--------- bonobo/util/python.py | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index 01441e4..2d13ab4 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -85,7 +85,6 @@ def execute( writer_option=None, option=None, filter=None, - do_print=False ): reader_factory, reader_option = resolve_factory(reader, input, READER, (option or []) + (reader_option or [])) @@ -121,7 +120,7 @@ def register(parser): parser.add_argument( '--' + WRITER, '-w', - help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong.' + help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).' ) parser.add_argument( '--filter', @@ -130,13 +129,6 @@ def register(parser): action='append', help='Add a filter between input and output', ) - parser.add_argument( - '--print', - '-p', - dest='do_print', - action='store_true', - help='Replace the output by a pretty printer.', - ) parser.add_argument( '--option', '-O', diff --git a/bonobo/util/python.py b/bonobo/util/python.py index e55c24c..8648f16 100644 --- a/bonobo/util/python.py +++ b/bonobo/util/python.py @@ -11,7 +11,7 @@ class _RequiredModule: class _RequiredModulesRegistry(dict): @property def pathname(self): - return os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.stack()[1][0]))) + return os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.stack()[2][0]))) def require(self, name): if name not in self: