From 2898902ebde81500c5dccc6d2cda259b97312413 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 12 Oct 2017 19:01:35 +0200 Subject: [PATCH] [cli] adds ability to override reader/writer options from cli convert. --- bonobo/commands/convert.py | 64 ++++++++++++++++++++++++------- bonobo/commands/util/__init__.py | 0 bonobo/commands/util/arguments.py | 26 +++++++++++++ bonobo/nodes/basics.py | 15 +++++++- bonobo/nodes/io/base.py | 1 + 5 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 bonobo/commands/util/__init__.py create mode 100644 bonobo/commands/util/arguments.py diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index 17b98c2..f02e8ab 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -1,7 +1,10 @@ import mimetypes import os +import re + import bonobo +from bonobo.commands.util.arguments import parse_variable_argument SHORTCUTS = { 'csv': 'text/csv', @@ -23,7 +26,7 @@ READER = 'reader' WRITER = 'writer' -def resolve_factory(name, filename, factory_type): +def resolve_factory(name, filename, factory_type, options=None): """ Try to resolve which transformation factory to use for this filename. User eventually provided a name, which has priority, otherwise we try to detect it using the mimetype detection on filename. @@ -42,6 +45,12 @@ def resolve_factory(name, filename, factory_type): if _ext in SHORTCUTS: name = SHORTCUTS[_ext] + if options: + options = dict(map(parse_variable_argument, options)) + + else: + options = dict() + if not name in REGISTRY: raise RuntimeError( 'Could not resolve {factory_type} factory for {filename} ({name}). Try providing it explicitely using -{opt} .'. @@ -49,19 +58,22 @@ def resolve_factory(name, filename, factory_type): ) if factory_type == READER: - return REGISTRY[name][0] + return REGISTRY[name][0], options elif factory_type == WRITER: - return REGISTRY[name][1] + return REGISTRY[name][1], options else: raise ValueError('Invalid factory type.') def execute(input, output, reader=None, reader_options=None, writer=None, writer_options=None, options=None): - reader = resolve_factory(reader, input, READER)(input) - writer = resolve_factory(writer, output, WRITER)(output) + reader_factory, reader_options = resolve_factory(reader, input, READER, (options or []) + (reader_options or [])) + writer_factory, writer_options = resolve_factory(writer, output, WRITER, (options or []) + (writer_options or [])) graph = bonobo.Graph() - graph.add_chain(reader, writer) + graph.add_chain( + reader_factory(input, **reader_options), + writer_factory(output, **writer_options), + ) return bonobo.run( graph, services={ @@ -71,11 +83,37 @@ def execute(input, output, reader=None, reader_options=None, writer=None, writer def register(parser): - parser.add_argument('input') - parser.add_argument('output') - parser.add_argument('--' + READER, '-r') - parser.add_argument('--' + WRITER, '-w') - # parser.add_argument('--reader-option', '-ro', dest='reader_options') - # parser.add_argument('--writer-option', '-wo', dest='writer_options') - # parser.add_argument('--option', '-o', dest='options') + parser.add_argument('input', help='Input filename.') + parser.add_argument('output', help='Output filename.') + parser.add_argument( + '--' + READER, + '-r', + help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.' + ) + parser.add_argument( + '--' + WRITER, + '-w', + help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong.' + ) + parser.add_argument( + '--option', + '-O', + dest='options', + action='append', + help='Add a named option to both reader and writer factories (i.e. foo="bar").' + ) + parser.add_argument( + '--' + READER + '-option', + '-' + READER[0].upper(), + dest=READER + '_options', + action='append', + help='Add a named option to the reader factory.' + ) + parser.add_argument( + '--' + WRITER + '-option', + '-' + WRITER[0].upper(), + dest=WRITER + '_options', + action='append', + help='Add a named option to the writer factory.' + ) return execute diff --git a/bonobo/commands/util/__init__.py b/bonobo/commands/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bonobo/commands/util/arguments.py b/bonobo/commands/util/arguments.py new file mode 100644 index 0000000..435c6f5 --- /dev/null +++ b/bonobo/commands/util/arguments.py @@ -0,0 +1,26 @@ +import json + + +def parse_variable_argument(arg): + try: + key, val = arg.split('=', 1) + except ValueError: + return arg, True + + try: + val = json.loads(val) + except json.JSONDecodeError: + pass + + return key, val + + +def test_parse_variable_argument(): + assert parse_variable_argument('foo=bar') == ('foo', 'bar') + assert parse_variable_argument('foo="bar"') == ('foo', 'bar') + assert parse_variable_argument('sep=";"') == ('sep', ';') + assert parse_variable_argument('foo') == ('foo', True) + + +if __name__ == '__main__': + test_parse_var() diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index e23dd05..b346404 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -70,7 +70,17 @@ def _count_counter(self, context): context.send(Bag(counter._value)) +def _shorten(s, w): + if w and len(s) > w: + s = s[0:w - 3] + '...' + return s + + class PrettyPrinter(Configurable): + max_width = Option(int, required=False, __doc__=''' + If set, truncates the output values longer than this to this width. + ''') + def call(self, *args, **kwargs): formater = self._format_quiet if settings.QUIET.get() else self._format_console @@ -82,7 +92,10 @@ class PrettyPrinter(Configurable): def _format_console(self, i, item, value): return ' '.join( - ((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL) + ( + (' ' if i else '•'), str(item), '=', _shorten(str(value).strip(), + self.max_width).replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL + ) ) diff --git a/bonobo/nodes/io/base.py b/bonobo/nodes/io/base.py index 496a0e8..3cecb70 100644 --- a/bonobo/nodes/io/base.py +++ b/bonobo/nodes/io/base.py @@ -50,6 +50,7 @@ class FileHandler(Configurable): eol = Option(str, default='\n') # type: str mode = Option(str) # type: str encoding = Option(str, default='utf-8') # type: str + fs = Service('fs') # type: str @ContextProcessor