Simpler package generation using cookiecutter, stdout buffering for consoleplugin

Projectfile

@@ -12,12 +12,12 @@ author_email = 'romain@dorgueil.net'
 
 enable_features = {
     'make',
-    'sphinx',
+    'sphinx',  # should install sphinx
-    'pytest',
+    'pytest',  # should install pytest/pytest-cov/coverage
     'git',
     'pylint',
     'python',
-    'yapf',
+    'yapf',  # should install yapf
 }
 
 # stricts deendencies in requirements.txt
@@ -30,10 +30,6 @@ install_requires = [
 ]
 
 extras_require = {
-    'jupyter': [
-        'jupyter >=1.0,<1.1',
-        'ipywidgets >=6.0.0.beta5'
-    ],
     'dev': [
         'coverage >=4,<5',
         'pylint >=1,<2',
@@ -41,9 +37,12 @@ extras_require = {
         'pytest-cov >=2,<3',
         'pytest-timeout >=1,<2',
         'sphinx',
-        'sphinx_rtd_theme',
         'yapf',
     ],
+    'jupyter': [
+        'jupyter >=1.0,<1.1',
+        'ipywidgets >=6.0.0.beta5'
+    ],
 }
 
 data_files = [
@@ -63,9 +62,6 @@ entry_points = {
         'run = bonobo.commands.run:register',
         'version = bonobo.commands.version:register',
     ],
-    'edgy.project.features': [
-        'bonobo = bonobo.ext.edgy.project.feature:BonoboFeature'
-    ]
 }
 
 @listen('edgy.project.feature.make.on_generate', priority=10)

bonobo/_api.py

@@ -1,12 +1,11 @@
 from bonobo.structs import Bag, Graph, Token
 from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \
-    PrettyPrint, Tee, count, identity, noop, pprint
+    PrettyPrinter, Tee, count, identity, noop, pprint
 from bonobo.strategies import create_strategy
 from bonobo.util.objects import get_name
 
 __all__ = []
 
-
 def register_api(x, __all__=__all__):
     __all__.append(get_name(x))
     return x
@@ -98,7 +97,7 @@ register_api_group(
     JsonReader,
     JsonWriter,
     Limit,
-    PrettyPrint,
+    PrettyPrinter,
     Tee,
     count,
     identity,

bonobo/commands/init.py

@@ -1,16 +1,17 @@
 import os
 
 
-def execute():
+def execute(name):
     try:
-        from edgy.project.__main__ import handle_init
+        from cookiecutter.main import cookiecutter
     except ImportError as exc:
         raise ImportError(
-            'You must install "edgy.project" to use this command.\n\n $ pip install edgy.project\n'
+            'You must install "cookiecutter" to use this command.\n\n $ pip install edgy.project\n'
         ) from exc
 
-    return handle_init(os.path.join(os.getcwd(), 'Projectfile'))
+    return cookiecutter('https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True)
 
 
 def register(parser):
+    parser.add_argument('name')
     return execute

bonobo/commands/version.py

@@ -1,9 +1,31 @@
 import bonobo
+from bonobo.util.packages import bonobo_packages
 
 
-def execute():
+def format_version(mod, *, name=None, quiet=False):
-    print('{} v.{}'.format(bonobo.__name__, bonobo.__version__))
+    return ('{name} {version}' if quiet else '{name} v.{version} (in {location})').format(
+        name=name or mod.__name__,
+        version=mod.__version__,
+        location=bonobo_packages[name or mod.__name__].location
+    )
+
+
+def execute(all=False, quiet=False):
+    print(format_version(bonobo, quiet=quiet))
+    if all:
+        for name in sorted(bonobo_packages):
+            if name != 'bonobo':
+                try:
+                    mod = __import__(name.replace('-', '_'))
+                    try:
+                        print(format_version(mod, name=name, quiet=quiet))
+                    except Exception as exc:
+                        print('{} ({})'.format(name, exc))
+                except ImportError as exc:
+                    print('{} is not importable ({}).'.format(name, exc))
 
 
 def register(parser):
+    parser.add_argument('--all', '-a', action='store_true')
+    parser.add_argument('--quiet', '-q', action='store_true')
     return execute

bonobo/examples/nodes/slow.py

@@ -0,0 +1,17 @@
+import bonobo
+import time
+
+
+from bonobo.constants import NOT_MODIFIED
+
+
+def pause(*args, **kwargs):
+    time.sleep(0.1)
+    return NOT_MODIFIED
+
+
+graph = bonobo.Graph(
+    lambda: tuple(range(20)),
+    pause,
+    print,
+)

bonobo/ext/console.py

@@ -1,5 +1,6 @@
-import functools
+import io
 import sys
+from contextlib import redirect_stdout
 
 from colorama import Style, Fore
 
@@ -8,6 +9,21 @@ from bonobo.plugins import Plugin
 from bonobo.util.term import CLEAR_EOL, MOVE_CURSOR_UP
 
 
+class IOBuffer():
+    def __init__(self):
+        self.current = io.StringIO()
+        self.write = self.current.write
+
+    def switch(self):
+        previous = self.current
+        self.current = io.StringIO()
+        self.write = self.current.write
+        try:
+            return previous.getvalue()
+        finally:
+            previous.close()
+
+
 class ConsoleOutputPlugin(Plugin):
     """
     Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor
@@ -23,34 +39,30 @@ class ConsoleOutputPlugin(Plugin):
         self.prefix = ''
         self.counter = 0
         self._append_cache = ''
+        self.isatty = sys.stdout.isatty()
 
-    def _write(self, graph_context, rewind):
+        self._stdout = sys.stdout
-        if settings.PROFILE:
+        self.stdout = IOBuffer()
-            if self.counter % 10 and self._append_cache:
+        self.redirect_stdout = redirect_stdout(self.stdout)
-                append = self._append_cache
+        self.redirect_stdout.__enter__()
-            else:
-                self._append_cache = append = (
-                    ('Memory', '{0:.2f} Mb'.format(memory_usage())),
-                    # ('Total time', '{0} s'.format(execution_time(harness))),
-                )
-        else:
-            append = ()
-        self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind)
-        self.counter += 1
 
     def run(self):
-        if sys.stdout.isatty():
+        if self.isatty:
             self._write(self.context.parent, rewind=True)
         else:
             pass  # not a tty
 
     def finalize(self):
         self._write(self.context.parent, rewind=False)
+        self.redirect_stdout.__exit__(None, None, None)
 
-    @staticmethod
+    def write(self, context, prefix='', rewind=True, append=None):
-    def write(context, prefix='', rewind=True, append=None):
         t_cnt = len(context)
 
+        buffered = self.stdout.switch()
+        for line in buffered.split('\n')[:-1]:
+            print(line + CLEAR_EOL, file=sys.stderr)
+
         for i in context.graph.topologically_sorted_indexes:
             node = context[i]
             name_suffix = '({})'.format(i) if settings.DEBUG else ''
@@ -68,7 +80,7 @@ class ConsoleOutputPlugin(Plugin):
                         Style.RESET_ALL, ' ',
                     )
                 )
-            print(prefix + _line + '\033[0K')
+            print(prefix + _line + '\033[0K', file=sys.stderr)
 
         if append:
             # todo handle multiline
@@ -78,16 +90,30 @@ class ConsoleOutputPlugin(Plugin):
                         ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v)
                                           for k, v in append), CLEAR_EOL
                     )
-                )
+                ), file=sys.stderr
             )
             t_cnt += 1
 
         if rewind:
-            print(CLEAR_EOL)
+            print(CLEAR_EOL, file=sys.stderr)
-            print(MOVE_CURSOR_UP(t_cnt + 2))
+            print(MOVE_CURSOR_UP(t_cnt + 2), file=sys.stderr)
+
+    def _write(self, graph_context, rewind):
+        if settings.PROFILE:
+            if self.counter % 10 and self._append_cache:
+                append = self._append_cache
+            else:
+                self._append_cache = append = (
+                    ('Memory', '{0:.2f} Mb'.format(memory_usage())),
+                    # ('Total time', '{0} s'.format(execution_time(harness))),
+                )
+        else:
+            append = ()
+        self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind)
+        self.counter += 1
 
 
 def memory_usage():
     import os, psutil
     process = psutil.Process(os.getpid())
-    return process.memory_info()[0] / float(2**20)
+    return process.memory_info()[0] / float(2 ** 20)

bonobo/ext/edgy/project/feature.py

@@ -1,22 +0,0 @@
-try:
-    import edgy.project
-except ImportError as e:
-    import logging
-
-    logging.exception('You must install edgy.project to use this.')
-
-import os
-
-from edgy.project.events import subscribe
-from edgy.project.feature import Feature, SUPPORT_PRIORITY
-
-
-class BonoboFeature(Feature):
-    requires = {'python'}
-
-    @subscribe('edgy.project.on_start', priority=SUPPORT_PRIORITY)
-    def on_start(self, event):
-        package_path = event.setup['name'].replace('.', os.sep)
-
-        for file in ('example_graph'):
-            self.render_file(os.path.join(package_path, file + '.py'), os.path.join('tornado', file + '.py.j2'))

bonobo/nodes/basics.py

@@ -18,7 +18,7 @@ __all__ = [
     'Tee',
     'count',
     'pprint',
-    'PrettyPrint',
+    'PrettyPrinter',
     'noop',
 ]
 
@@ -85,7 +85,8 @@ class PrettyPrinter(Configurable):
         return ' '.join(((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL))
 
 
-pprint = Tee(_pprint)
+pprint = PrettyPrinter()
+pprint.__name__ = 'pprint'
 
 
 def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):

bonobo/util/packages.py

@@ -0,0 +1,8 @@
+import pkg_resources
+from packaging.utils import canonicalize_name
+
+bonobo_packages = {}
+for p in pkg_resources.working_set:
+    name = canonicalize_name(p.project_name)
+    if name.startswith('bonobo'):
+        bonobo_packages[name] = p