From eacf52aaf6d0ab8e45a7e40ed265c5afa64fc712 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 14:55:25 +0200 Subject: [PATCH] Simpler package generation using cookiecutter, stdout buffering for consoleplugin --- Projectfile | 18 +++----- bonobo/_api.py | 5 +-- bonobo/commands/init.py | 9 ++-- bonobo/commands/version.py | 26 ++++++++++- bonobo/examples/nodes/slow.py | 17 +++++++ bonobo/ext/console.py | 70 ++++++++++++++++++++--------- bonobo/ext/edgy/__init__.py | 0 bonobo/ext/edgy/project/__init__.py | 0 bonobo/ext/edgy/project/feature.py | 22 --------- bonobo/nodes/basics.py | 5 ++- bonobo/util/packages.py | 8 ++++ 11 files changed, 114 insertions(+), 66 deletions(-) create mode 100644 bonobo/examples/nodes/slow.py delete mode 100644 bonobo/ext/edgy/__init__.py delete mode 100644 bonobo/ext/edgy/project/__init__.py delete mode 100644 bonobo/ext/edgy/project/feature.py create mode 100644 bonobo/util/packages.py diff --git a/Projectfile b/Projectfile index 47844ee..d8fcf44 100644 --- a/Projectfile +++ b/Projectfile @@ -12,12 +12,12 @@ author_email = 'romain@dorgueil.net' enable_features = { 'make', - 'sphinx', - 'pytest', + 'sphinx', # should install sphinx + 'pytest', # should install pytest/pytest-cov/coverage 'git', 'pylint', 'python', - 'yapf', + 'yapf', # should install yapf } # stricts deendencies in requirements.txt @@ -30,10 +30,6 @@ install_requires = [ ] extras_require = { - 'jupyter': [ - 'jupyter >=1.0,<1.1', - 'ipywidgets >=6.0.0.beta5' - ], 'dev': [ 'coverage >=4,<5', 'pylint >=1,<2', @@ -41,9 +37,12 @@ extras_require = { 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', - 'sphinx_rtd_theme', 'yapf', ], + 'jupyter': [ + 'jupyter >=1.0,<1.1', + 'ipywidgets >=6.0.0.beta5' + ], } data_files = [ @@ -63,9 +62,6 @@ entry_points = { 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register', ], - 'edgy.project.features': [ - 'bonobo = bonobo.ext.edgy.project.feature:BonoboFeature' - ] } @listen('edgy.project.feature.make.on_generate', priority=10) diff --git a/bonobo/_api.py b/bonobo/_api.py index ca5f363..049eba4 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,12 +1,11 @@ from bonobo.structs import Bag, Graph, Token from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \ - PrettyPrint, Tee, count, identity, noop, pprint + PrettyPrinter, Tee, count, identity, noop, pprint from bonobo.strategies import create_strategy from bonobo.util.objects import get_name __all__ = [] - def register_api(x, __all__=__all__): __all__.append(get_name(x)) return x @@ -98,7 +97,7 @@ register_api_group( JsonReader, JsonWriter, Limit, - PrettyPrint, + PrettyPrinter, Tee, count, identity, diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index 55c4b6b..d13a21f 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -1,16 +1,17 @@ import os -def execute(): +def execute(name): try: - from edgy.project.__main__ import handle_init + from cookiecutter.main import cookiecutter except ImportError as exc: raise ImportError( - 'You must install "edgy.project" to use this command.\n\n $ pip install edgy.project\n' + 'You must install "cookiecutter" to use this command.\n\n $ pip install edgy.project\n' ) from exc - return handle_init(os.path.join(os.getcwd(), 'Projectfile')) + return cookiecutter('https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True) def register(parser): + parser.add_argument('name') return execute diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index 332286a..c6ac9e8 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -1,9 +1,31 @@ import bonobo +from bonobo.util.packages import bonobo_packages -def execute(): - print('{} v.{}'.format(bonobo.__name__, bonobo.__version__)) +def format_version(mod, *, name=None, quiet=False): + return ('{name} {version}' if quiet else '{name} v.{version} (in {location})').format( + name=name or mod.__name__, + version=mod.__version__, + location=bonobo_packages[name or mod.__name__].location + ) + + +def execute(all=False, quiet=False): + print(format_version(bonobo, quiet=quiet)) + if all: + for name in sorted(bonobo_packages): + if name != 'bonobo': + try: + mod = __import__(name.replace('-', '_')) + try: + print(format_version(mod, name=name, quiet=quiet)) + except Exception as exc: + print('{} ({})'.format(name, exc)) + except ImportError as exc: + print('{} is not importable ({}).'.format(name, exc)) def register(parser): + parser.add_argument('--all', '-a', action='store_true') + parser.add_argument('--quiet', '-q', action='store_true') return execute diff --git a/bonobo/examples/nodes/slow.py b/bonobo/examples/nodes/slow.py new file mode 100644 index 0000000..703ebc2 --- /dev/null +++ b/bonobo/examples/nodes/slow.py @@ -0,0 +1,17 @@ +import bonobo +import time + + +from bonobo.constants import NOT_MODIFIED + + +def pause(*args, **kwargs): + time.sleep(0.1) + return NOT_MODIFIED + + +graph = bonobo.Graph( + lambda: tuple(range(20)), + pause, + print, +) diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index d454bcd..6a3ef8e 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -1,5 +1,6 @@ -import functools +import io import sys +from contextlib import redirect_stdout from colorama import Style, Fore @@ -8,6 +9,21 @@ from bonobo.plugins import Plugin from bonobo.util.term import CLEAR_EOL, MOVE_CURSOR_UP +class IOBuffer(): + def __init__(self): + self.current = io.StringIO() + self.write = self.current.write + + def switch(self): + previous = self.current + self.current = io.StringIO() + self.write = self.current.write + try: + return previous.getvalue() + finally: + previous.close() + + class ConsoleOutputPlugin(Plugin): """ Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor @@ -23,34 +39,30 @@ class ConsoleOutputPlugin(Plugin): self.prefix = '' self.counter = 0 self._append_cache = '' + self.isatty = sys.stdout.isatty() - def _write(self, graph_context, rewind): - if settings.PROFILE: - if self.counter % 10 and self._append_cache: - append = self._append_cache - else: - self._append_cache = append = ( - ('Memory', '{0:.2f} Mb'.format(memory_usage())), - # ('Total time', '{0} s'.format(execution_time(harness))), - ) - else: - append = () - self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind) - self.counter += 1 + self._stdout = sys.stdout + self.stdout = IOBuffer() + self.redirect_stdout = redirect_stdout(self.stdout) + self.redirect_stdout.__enter__() def run(self): - if sys.stdout.isatty(): + if self.isatty: self._write(self.context.parent, rewind=True) else: pass # not a tty def finalize(self): self._write(self.context.parent, rewind=False) + self.redirect_stdout.__exit__(None, None, None) - @staticmethod - def write(context, prefix='', rewind=True, append=None): + def write(self, context, prefix='', rewind=True, append=None): t_cnt = len(context) + buffered = self.stdout.switch() + for line in buffered.split('\n')[:-1]: + print(line + CLEAR_EOL, file=sys.stderr) + for i in context.graph.topologically_sorted_indexes: node = context[i] name_suffix = '({})'.format(i) if settings.DEBUG else '' @@ -68,7 +80,7 @@ class ConsoleOutputPlugin(Plugin): Style.RESET_ALL, ' ', ) ) - print(prefix + _line + '\033[0K') + print(prefix + _line + '\033[0K', file=sys.stderr) if append: # todo handle multiline @@ -78,16 +90,30 @@ class ConsoleOutputPlugin(Plugin): ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), CLEAR_EOL ) - ) + ), file=sys.stderr ) t_cnt += 1 if rewind: - print(CLEAR_EOL) - print(MOVE_CURSOR_UP(t_cnt + 2)) + print(CLEAR_EOL, file=sys.stderr) + print(MOVE_CURSOR_UP(t_cnt + 2), file=sys.stderr) + + def _write(self, graph_context, rewind): + if settings.PROFILE: + if self.counter % 10 and self._append_cache: + append = self._append_cache + else: + self._append_cache = append = ( + ('Memory', '{0:.2f} Mb'.format(memory_usage())), + # ('Total time', '{0} s'.format(execution_time(harness))), + ) + else: + append = () + self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind) + self.counter += 1 def memory_usage(): import os, psutil process = psutil.Process(os.getpid()) - return process.memory_info()[0] / float(2**20) \ No newline at end of file + return process.memory_info()[0] / float(2 ** 20) diff --git a/bonobo/ext/edgy/__init__.py b/bonobo/ext/edgy/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bonobo/ext/edgy/project/__init__.py b/bonobo/ext/edgy/project/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bonobo/ext/edgy/project/feature.py b/bonobo/ext/edgy/project/feature.py deleted file mode 100644 index 5a01f97..0000000 --- a/bonobo/ext/edgy/project/feature.py +++ /dev/null @@ -1,22 +0,0 @@ -try: - import edgy.project -except ImportError as e: - import logging - - logging.exception('You must install edgy.project to use this.') - -import os - -from edgy.project.events import subscribe -from edgy.project.feature import Feature, SUPPORT_PRIORITY - - -class BonoboFeature(Feature): - requires = {'python'} - - @subscribe('edgy.project.on_start', priority=SUPPORT_PRIORITY) - def on_start(self, event): - package_path = event.setup['name'].replace('.', os.sep) - - for file in ('example_graph'): - self.render_file(os.path.join(package_path, file + '.py'), os.path.join('tornado', file + '.py.j2')) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index bbf6ab5..1242cb2 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -18,7 +18,7 @@ __all__ = [ 'Tee', 'count', 'pprint', - 'PrettyPrint', + 'PrettyPrinter', 'noop', ] @@ -85,7 +85,8 @@ class PrettyPrinter(Configurable): return ' '.join(((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)) -pprint = Tee(_pprint) +pprint = PrettyPrinter() +pprint.__name__ = 'pprint' def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True): diff --git a/bonobo/util/packages.py b/bonobo/util/packages.py new file mode 100644 index 0000000..e4de1b6 --- /dev/null +++ b/bonobo/util/packages.py @@ -0,0 +1,8 @@ +import pkg_resources +from packaging.utils import canonicalize_name + +bonobo_packages = {} +for p in pkg_resources.working_set: + name = canonicalize_name(p.project_name) + if name.startswith('bonobo'): + bonobo_packages[name] = p