From 0ac9e05956aa9b65f223922f2f5362fbc357ddfe Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 29 May 2017 05:11:07 -0700 Subject: [PATCH 01/11] Update sphinx from 1.6.1 to 1.6.2 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 18f7807..1f03666 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ pytz==2017.2 requests==2.16.5 six==1.10.0 snowballstemmer==1.2.1 -sphinx==1.6.1 +sphinx==1.6.2 sphinxcontrib-websupport==1.0.1 typing==3.6.1 urllib3==1.21.1 From 7157e890afa0652e39801443fc6b458e2de6790f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 29 May 2017 16:15:55 -0700 Subject: [PATCH 02/11] Update requests from 2.16.5 to 2.17.3 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 18f7807..8b01e64 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,7 +15,7 @@ pytest-cov==2.5.1 pytest-timeout==1.2.0 pytest==3.1.0 pytz==2017.2 -requests==2.16.5 +requests==2.17.3 six==1.10.0 snowballstemmer==1.2.1 sphinx==1.6.1 From b2e3daf8c305c6e373065347bff36dbcb28270b3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 29 May 2017 16:15:57 -0700 Subject: [PATCH 03/11] Update requests from 2.16.5 to 2.17.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e9bdffb..6845a63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ idna==2.5 pbr==3.0.1 psutil==5.2.2 pytz==2017.2 -requests==2.16.5 +requests==2.17.3 six==1.10.0 stevedore==1.21.0 urllib3==1.21.1 From 0a429ab079bc9f65eb4befb3a3bc60b129cbb23d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 30 May 2017 06:33:59 -0700 Subject: [PATCH 04/11] Update stevedore from 1.21.0 to 1.22.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e9bdffb..689f472 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,5 @@ psutil==5.2.2 pytz==2017.2 requests==2.16.5 six==1.10.0 -stevedore==1.21.0 +stevedore==1.22.0 urllib3==1.21.1 From 8963ded2d1202f33785ce40a85c44ed68f2a9647 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 31 May 2017 15:16:55 +0200 Subject: [PATCH 05/11] Update pytest from 3.1.0 to 3.1.1 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 18f7807..479a21a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,7 @@ py==1.4.33 pygments==2.2.0 pytest-cov==2.5.1 pytest-timeout==1.2.0 -pytest==3.1.0 +pytest==3.1.1 pytz==2017.2 requests==2.16.5 six==1.10.0 From 1d070636e9dc1df8cd142ff58938f12e8c4dec05 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 31 May 2017 15:16:56 +0200 Subject: [PATCH 06/11] Update requests from 2.16.5 to 2.17.3 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 479a21a..f67bee1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,7 +15,7 @@ pytest-cov==2.5.1 pytest-timeout==1.2.0 pytest==3.1.1 pytz==2017.2 -requests==2.16.5 +requests==2.17.3 six==1.10.0 snowballstemmer==1.2.1 sphinx==1.6.1 From 99f9952c7cbf3fea1204eaea69521ecc124cd59c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 31 May 2017 15:16:58 +0200 Subject: [PATCH 07/11] Update requests from 2.16.5 to 2.17.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e9bdffb..6845a63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ idna==2.5 pbr==3.0.1 psutil==5.2.2 pytz==2017.2 -requests==2.16.5 +requests==2.17.3 six==1.10.0 stevedore==1.21.0 urllib3==1.21.1 From bd6eddc48e1c8e5a4d1bf4529628c326b0d6700d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 31 May 2017 15:17:49 +0200 Subject: [PATCH 08/11] Update sphinx from 1.6.1 to 1.6.2 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f67bee1..c7a14c6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ pytz==2017.2 requests==2.17.3 six==1.10.0 snowballstemmer==1.2.1 -sphinx==1.6.1 +sphinx==1.6.2 sphinxcontrib-websupport==1.0.1 typing==3.6.1 urllib3==1.21.1 From 203843f9943b25e72804f720fb7fa2df8148fcdd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 31 May 2017 15:17:51 +0200 Subject: [PATCH 09/11] Update stevedore from 1.21.0 to 1.22.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6845a63..8e999c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,5 @@ psutil==5.2.2 pytz==2017.2 requests==2.17.3 six==1.10.0 -stevedore==1.21.0 +stevedore==1.22.0 urllib3==1.21.1 From 4b0015706e1f73735e8362ffa94bdb307eee0946 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 31 May 2017 19:32:59 +0200 Subject: [PATCH 10/11] Update pytest from 3.1.0 to 3.1.1 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d2deed2..c7a14c6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,7 @@ py==1.4.33 pygments==2.2.0 pytest-cov==2.5.1 pytest-timeout==1.2.0 -pytest==3.1.0 +pytest==3.1.1 pytz==2017.2 requests==2.17.3 six==1.10.0 From 88d46a724dda39532fb8a73f80d01da9c047f237 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 31 May 2017 22:06:08 +0200 Subject: [PATCH 11/11] [cli] adds python logging facility configuration. --- bonobo/commands/__init__.py | 16 ++++++-- bonobo/commands/init.py | 3 -- bonobo/commands/run.py | 20 ++++------ bonobo/commands/version.py | 8 ++-- bonobo/logging.py | 75 +++++++++++++++++++++++++++++++++++++ bonobo/settings.py | 11 ++++-- bonobo/util/iterators.py | 16 +++++++- 7 files changed, 122 insertions(+), 27 deletions(-) create mode 100644 bonobo/logging.py diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index feae672..59e6dfb 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -1,12 +1,13 @@ import argparse -import logging -from stevedore import ExtensionManager +from bonobo import logging, settings + +logger = logging.get_logger() def entrypoint(args=None): - logging.basicConfig() parser = argparse.ArgumentParser() + parser.add_argument('--debug', '-D', action='store_true') subparsers = parser.add_subparsers(dest='command') subparsers.required = True @@ -18,10 +19,17 @@ def entrypoint(args=None): parser = subparsers.add_parser(ext.name) commands[ext.name] = ext.plugin(parser) except Exception: - logging.exception('Error while loading command {}.'.format(ext.name)) + logger.exception('Error while loading command {}.'.format(ext.name)) + from stevedore import ExtensionManager mgr = ExtensionManager(namespace='bonobo.commands') mgr.map(register_extension) args = parser.parse_args(args).__dict__ + if args.pop('debug', False): + settings.DEBUG = True + settings.LOGGING_LEVEL = logging.DEBUG + logging.set_level(settings.LOGGING_LEVEL) + + logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args)) commands[args.pop('command')](**args) diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index ad3c52a..81af38c 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -1,6 +1,3 @@ -import os - - def execute(name): try: from cookiecutter.main import cookiecutter diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 8bb7b10..b2e1bcc 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -1,10 +1,4 @@ -import importlib import os -import runpy - -import pip - -import bonobo DEFAULT_SERVICES_FILENAME = '_services.py' DEFAULT_SERVICES_ATTR = 'get_services' @@ -12,7 +6,6 @@ DEFAULT_SERVICES_ATTR = 'get_services' DEFAULT_GRAPH_FILENAME = '__main__.py' DEFAULT_GRAPH_ATTR = 'get_graph' - def get_default_services(filename, services=None): dirname = os.path.dirname(filename) services_filename = os.path.join(dirname, DEFAULT_SERVICES_FILENAME) @@ -33,7 +26,8 @@ def get_default_services(filename, services=None): def execute(filename, module, install=False, quiet=False, verbose=False): - from bonobo import settings + import runpy + from bonobo import Graph, run, settings if quiet: settings.QUIET = True @@ -44,6 +38,8 @@ def execute(filename, module, install=False, quiet=False, verbose=False): if filename: if os.path.isdir(filename): if install: + import importlib + import pip requirements = os.path.join(filename, 'requirements.txt') pip.main(['install', '-r', requirements]) # Some shenanigans to be sure everything is importable after this, especially .egg-link files which @@ -62,7 +58,7 @@ def execute(filename, module, install=False, quiet=False, verbose=False): else: raise RuntimeError('UNEXPECTED: argparse should not allow this.') - graphs = dict((k, v) for k, v in context.items() if isinstance(v, bonobo.Graph)) + graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph)) assert len(graphs) == 1, ( 'Having zero or more than one graph definition in one file is unsupported for now, ' @@ -73,7 +69,7 @@ def execute(filename, module, install=False, quiet=False, verbose=False): # todo if console and not quiet, then add the console plugin # todo when better console plugin, add it if console and just disable display - return bonobo.run( + return run( graph, plugins=[], services=get_default_services( @@ -82,8 +78,8 @@ def execute(filename, module, install=False, quiet=False, verbose=False): ) -def register_generic_run_arguments(parser): - source_group = parser.add_mutually_exclusive_group(required=True) +def register_generic_run_arguments(parser, required=True): + source_group = parser.add_mutually_exclusive_group(required=required) source_group.add_argument('filename', nargs='?', type=str) source_group.add_argument('--module', '-m', type=str) return parser diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index bfa03a7..6d4f3e7 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -1,8 +1,5 @@ -import bonobo -from bonobo.util.pkgs import bonobo_packages - - def format_version(mod, *, name=None, quiet=False): + from bonobo.util.pkgs import bonobo_packages args = { 'name': name or mod.__name__, 'version': mod.__version__, @@ -20,6 +17,9 @@ def format_version(mod, *, name=None, quiet=False): def execute(all=False, quiet=False): + import bonobo + from bonobo.util.pkgs import bonobo_packages + print(format_version(bonobo, quiet=quiet)) if all: for name in sorted(bonobo_packages): diff --git a/bonobo/logging.py b/bonobo/logging.py new file mode 100644 index 0000000..1561089 --- /dev/null +++ b/bonobo/logging.py @@ -0,0 +1,75 @@ +import logging +import sys +import textwrap +from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING + +from colorama import Fore, Style + +from bonobo import settings +from bonobo.util.term import CLEAR_EOL + + +def get_format(): + yield '{b}[%(fg)s%(levelname)s{b}][{w}' + yield '{b}][{w}'.join(('%(spent)04d', '%(name)s')) + yield '{b}]' + yield ' %(fg)s%(message)s{r}' + yield CLEAR_EOL + + +colors = { + 'b': Fore.BLACK, + 'w': Fore.LIGHTBLACK_EX, + 'r': Style.RESET_ALL, +} +format = (''.join(get_format())).format(**colors) + + +class Filter(logging.Filter): + def filter(self, record): + record.spent = record.relativeCreated // 1000 + if record.levelname == 'DEBG': + record.fg = Fore.LIGHTBLACK_EX + elif record.levelname == 'INFO': + record.fg = Fore.LIGHTWHITE_EX + elif record.levelname == 'WARN': + record.fg = Fore.LIGHTYELLOW_EX + elif record.levelname == 'ERR ': + record.fg = Fore.LIGHTRED_EX + elif record.levelname == 'CRIT': + record.fg = Fore.RED + else: + record.fg = Fore.LIGHTWHITE_EX + return True + + +class Formatter(logging.Formatter): + def formatException(self, ei): + tb = super().formatException(ei) + return textwrap.indent(tb, Fore.BLACK + ' | ' + Fore.WHITE) + + +def setup(level): + logging.addLevelName(DEBUG, 'DEBG') + logging.addLevelName(INFO, 'INFO') + logging.addLevelName(WARNING, 'WARN') + logging.addLevelName(ERROR, 'ERR ') + logging.addLevelName(CRITICAL, 'CRIT') + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(Formatter(format)) + handler.addFilter(Filter()) + root = logging.getLogger() + root.addHandler(handler) + root.setLevel(level) + + +def set_level(level): + logging.getLogger().setLevel(level) + + +def get_logger(name='bonobo'): + return logging.getLogger(name) + + +# Setup formating and level. +setup(level=settings.LOGGING_LEVEL) diff --git a/bonobo/settings.py b/bonobo/settings.py index d956b2c..dda7ba7 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -1,5 +1,7 @@ import os +import logging + def to_bool(s): if len(s): @@ -10,13 +12,16 @@ def to_bool(s): # Debug/verbose mode. -DEBUG = to_bool(os.environ.get('BONOBO_DEBUG', 'f')) +DEBUG = to_bool(os.environ.get('DEBUG', 'f')) # Profile mode. -PROFILE = to_bool(os.environ.get('BONOBO_PROFILE', 'f')) +PROFILE = to_bool(os.environ.get('PROFILE', 'f')) # Quiet mode. -QUIET = to_bool(os.environ.get('BONOBO_QUIET', 'f')) +QUIET = to_bool(os.environ.get('QUIET', 'f')) + +# Logging level. +LOGGING_LEVEL = logging.DEBUG if DEBUG else logging.INFO def check(): diff --git a/bonobo/util/iterators.py b/bonobo/util/iterators.py index 142b35a..ae39a49 100644 --- a/bonobo/util/iterators.py +++ b/bonobo/util/iterators.py @@ -1,4 +1,5 @@ """ Iterator utilities. """ +import functools def force_iterator(mixed): @@ -20,7 +21,20 @@ def force_iterator(mixed): def ensure_tuple(tuple_or_mixed): if isinstance(tuple_or_mixed, tuple): return tuple_or_mixed - return (tuple_or_mixed, ) + return (tuple_or_mixed,) + + +def tuplize(generator): + """ Takes a generator and make it a tuple-returning function. As a side + effect, it can also decorate any iterator-returning function to force + return value to be a tuple. + """ + + @functools.wraps(generator) + def tuplized(*args, **kwargs): + return tuple(generator(*args, **kwargs)) + + return tuplized def iter_if_not_sequence(mixed):