From d489eb2c4ced23a4b0bf8f0d5d08e54edea7fb1c Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 25 May 2017 15:44:18 +0200 Subject: [PATCH 1/9] [docs] tldr on how to create a feature branch and pull request it. --- docs/contribute/index.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/contribute/index.rst b/docs/contribute/index.rst index 3c3f6e9..690cab7 100644 --- a/docs/contribute/index.rst +++ b/docs/contribute/index.rst @@ -13,6 +13,26 @@ contributions have less value, all contributions are very important. * You can enhance tests. * etc. +tl;dr +::::: + +1. Fork the github repository + +.. code-block:: shell-session + + $ git clone https://github.com/python-bonobo/bonobo.git # change this to use your fork. + $ cd bonobo + $ git remote add upstream https://github.com/python-bonobo/bonobo.git + $ git fetch upstream + $ git checkout upstream/develop -b feature/my_awesome_feature + $ # code, code, code, test, doc, code, test ... + $ git commit -m '[topic] .... blaaaah ....' + $ git push origin feature/my_awesome_feature + +2. Open pull request +3. Rince, repeat + + Code-related contributions (including tests and examples) ::::::::::::::::::::::::::::::::::::::::::::::::::::::::: From 34aa357fd32fae66662d4d0b5a42186ebacf2073 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Thu, 25 May 2017 19:49:05 +0200 Subject: [PATCH 2/9] [misc] minor output tuning. --- bonobo/ext/console.py | 15 ++++++++++----- bonobo/nodes/basics.py | 11 ++++++++++- bonobo/settings.py | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index e15453b..d454bcd 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -21,16 +21,22 @@ class ConsoleOutputPlugin(Plugin): def initialize(self): self.prefix = '' + self.counter = 0 + self._append_cache = '' def _write(self, graph_context, rewind): if settings.PROFILE: - append = ( - ('Memory', '{0:.2f} Mb'.format(memory_usage())), - # ('Total time', '{0} s'.format(execution_time(harness))), - ) + if self.counter % 10 and self._append_cache: + append = self._append_cache + else: + self._append_cache = append = ( + ('Memory', '{0:.2f} Mb'.format(memory_usage())), + # ('Total time', '{0} s'.format(execution_time(harness))), + ) else: append = () self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind) + self.counter += 1 def run(self): if sys.stdout.isatty(): @@ -81,7 +87,6 @@ class ConsoleOutputPlugin(Plugin): print(MOVE_CURSOR_UP(t_cnt + 2)) -@functools.lru_cache(1) def memory_usage(): import os, psutil process = psutil.Process(os.getpid()) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 5ce550c..bbf6ab5 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -4,6 +4,7 @@ from pprint import pprint as _pprint import itertools from colorama import Fore, Style +from bonobo import settings from bonobo.config import Configurable, Option from bonobo.config.processors import ContextProcessor from bonobo.structs.bags import Bag @@ -72,8 +73,16 @@ def _count_counter(self, context): class PrettyPrinter(Configurable): def call(self, *args, **kwargs): + formater = self._format_quiet if settings.QUIET else self._format_console + for i, (item, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())): - print(' ' if i else '•', item, '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL) + print(formater(i, item, value)) + + def _format_quiet(self, i, item, value): + return ' '.join(((' ' if i else '-'), str(item), ':', str(value).strip())) + + def _format_console(self, i, item, value): + return ' '.join(((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)) pprint = Tee(_pprint) diff --git a/bonobo/settings.py b/bonobo/settings.py index 9f0fbf6..d956b2c 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -21,4 +21,4 @@ QUIET = to_bool(os.environ.get('BONOBO_QUIET', 'f')) def check(): if DEBUG and QUIET: - raise RuntimeError('I cannot be verbose and quiet at the same time.') \ No newline at end of file + raise RuntimeError('I cannot be verbose and quiet at the same time.') From eacf52aaf6d0ab8e45a7e40ed265c5afa64fc712 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 14:55:25 +0200 Subject: [PATCH 3/9] Simpler package generation using cookiecutter, stdout buffering for consoleplugin --- Projectfile | 18 +++----- bonobo/_api.py | 5 +-- bonobo/commands/init.py | 9 ++-- bonobo/commands/version.py | 26 ++++++++++- bonobo/examples/nodes/slow.py | 17 +++++++ bonobo/ext/console.py | 70 ++++++++++++++++++++--------- bonobo/ext/edgy/__init__.py | 0 bonobo/ext/edgy/project/__init__.py | 0 bonobo/ext/edgy/project/feature.py | 22 --------- bonobo/nodes/basics.py | 5 ++- bonobo/util/packages.py | 8 ++++ 11 files changed, 114 insertions(+), 66 deletions(-) create mode 100644 bonobo/examples/nodes/slow.py delete mode 100644 bonobo/ext/edgy/__init__.py delete mode 100644 bonobo/ext/edgy/project/__init__.py delete mode 100644 bonobo/ext/edgy/project/feature.py create mode 100644 bonobo/util/packages.py diff --git a/Projectfile b/Projectfile index 47844ee..d8fcf44 100644 --- a/Projectfile +++ b/Projectfile @@ -12,12 +12,12 @@ author_email = 'romain@dorgueil.net' enable_features = { 'make', - 'sphinx', - 'pytest', + 'sphinx', # should install sphinx + 'pytest', # should install pytest/pytest-cov/coverage 'git', 'pylint', 'python', - 'yapf', + 'yapf', # should install yapf } # stricts deendencies in requirements.txt @@ -30,10 +30,6 @@ install_requires = [ ] extras_require = { - 'jupyter': [ - 'jupyter >=1.0,<1.1', - 'ipywidgets >=6.0.0.beta5' - ], 'dev': [ 'coverage >=4,<5', 'pylint >=1,<2', @@ -41,9 +37,12 @@ extras_require = { 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', - 'sphinx_rtd_theme', 'yapf', ], + 'jupyter': [ + 'jupyter >=1.0,<1.1', + 'ipywidgets >=6.0.0.beta5' + ], } data_files = [ @@ -63,9 +62,6 @@ entry_points = { 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register', ], - 'edgy.project.features': [ - 'bonobo = bonobo.ext.edgy.project.feature:BonoboFeature' - ] } @listen('edgy.project.feature.make.on_generate', priority=10) diff --git a/bonobo/_api.py b/bonobo/_api.py index ca5f363..049eba4 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,12 +1,11 @@ from bonobo.structs import Bag, Graph, Token from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \ - PrettyPrint, Tee, count, identity, noop, pprint + PrettyPrinter, Tee, count, identity, noop, pprint from bonobo.strategies import create_strategy from bonobo.util.objects import get_name __all__ = [] - def register_api(x, __all__=__all__): __all__.append(get_name(x)) return x @@ -98,7 +97,7 @@ register_api_group( JsonReader, JsonWriter, Limit, - PrettyPrint, + PrettyPrinter, Tee, count, identity, diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index 55c4b6b..d13a21f 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -1,16 +1,17 @@ import os -def execute(): +def execute(name): try: - from edgy.project.__main__ import handle_init + from cookiecutter.main import cookiecutter except ImportError as exc: raise ImportError( - 'You must install "edgy.project" to use this command.\n\n $ pip install edgy.project\n' + 'You must install "cookiecutter" to use this command.\n\n $ pip install edgy.project\n' ) from exc - return handle_init(os.path.join(os.getcwd(), 'Projectfile')) + return cookiecutter('https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True) def register(parser): + parser.add_argument('name') return execute diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index 332286a..c6ac9e8 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -1,9 +1,31 @@ import bonobo +from bonobo.util.packages import bonobo_packages -def execute(): - print('{} v.{}'.format(bonobo.__name__, bonobo.__version__)) +def format_version(mod, *, name=None, quiet=False): + return ('{name} {version}' if quiet else '{name} v.{version} (in {location})').format( + name=name or mod.__name__, + version=mod.__version__, + location=bonobo_packages[name or mod.__name__].location + ) + + +def execute(all=False, quiet=False): + print(format_version(bonobo, quiet=quiet)) + if all: + for name in sorted(bonobo_packages): + if name != 'bonobo': + try: + mod = __import__(name.replace('-', '_')) + try: + print(format_version(mod, name=name, quiet=quiet)) + except Exception as exc: + print('{} ({})'.format(name, exc)) + except ImportError as exc: + print('{} is not importable ({}).'.format(name, exc)) def register(parser): + parser.add_argument('--all', '-a', action='store_true') + parser.add_argument('--quiet', '-q', action='store_true') return execute diff --git a/bonobo/examples/nodes/slow.py b/bonobo/examples/nodes/slow.py new file mode 100644 index 0000000..703ebc2 --- /dev/null +++ b/bonobo/examples/nodes/slow.py @@ -0,0 +1,17 @@ +import bonobo +import time + + +from bonobo.constants import NOT_MODIFIED + + +def pause(*args, **kwargs): + time.sleep(0.1) + return NOT_MODIFIED + + +graph = bonobo.Graph( + lambda: tuple(range(20)), + pause, + print, +) diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index d454bcd..6a3ef8e 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -1,5 +1,6 @@ -import functools +import io import sys +from contextlib import redirect_stdout from colorama import Style, Fore @@ -8,6 +9,21 @@ from bonobo.plugins import Plugin from bonobo.util.term import CLEAR_EOL, MOVE_CURSOR_UP +class IOBuffer(): + def __init__(self): + self.current = io.StringIO() + self.write = self.current.write + + def switch(self): + previous = self.current + self.current = io.StringIO() + self.write = self.current.write + try: + return previous.getvalue() + finally: + previous.close() + + class ConsoleOutputPlugin(Plugin): """ Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor @@ -23,34 +39,30 @@ class ConsoleOutputPlugin(Plugin): self.prefix = '' self.counter = 0 self._append_cache = '' + self.isatty = sys.stdout.isatty() - def _write(self, graph_context, rewind): - if settings.PROFILE: - if self.counter % 10 and self._append_cache: - append = self._append_cache - else: - self._append_cache = append = ( - ('Memory', '{0:.2f} Mb'.format(memory_usage())), - # ('Total time', '{0} s'.format(execution_time(harness))), - ) - else: - append = () - self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind) - self.counter += 1 + self._stdout = sys.stdout + self.stdout = IOBuffer() + self.redirect_stdout = redirect_stdout(self.stdout) + self.redirect_stdout.__enter__() def run(self): - if sys.stdout.isatty(): + if self.isatty: self._write(self.context.parent, rewind=True) else: pass # not a tty def finalize(self): self._write(self.context.parent, rewind=False) + self.redirect_stdout.__exit__(None, None, None) - @staticmethod - def write(context, prefix='', rewind=True, append=None): + def write(self, context, prefix='', rewind=True, append=None): t_cnt = len(context) + buffered = self.stdout.switch() + for line in buffered.split('\n')[:-1]: + print(line + CLEAR_EOL, file=sys.stderr) + for i in context.graph.topologically_sorted_indexes: node = context[i] name_suffix = '({})'.format(i) if settings.DEBUG else '' @@ -68,7 +80,7 @@ class ConsoleOutputPlugin(Plugin): Style.RESET_ALL, ' ', ) ) - print(prefix + _line + '\033[0K') + print(prefix + _line + '\033[0K', file=sys.stderr) if append: # todo handle multiline @@ -78,16 +90,30 @@ class ConsoleOutputPlugin(Plugin): ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), CLEAR_EOL ) - ) + ), file=sys.stderr ) t_cnt += 1 if rewind: - print(CLEAR_EOL) - print(MOVE_CURSOR_UP(t_cnt + 2)) + print(CLEAR_EOL, file=sys.stderr) + print(MOVE_CURSOR_UP(t_cnt + 2), file=sys.stderr) + + def _write(self, graph_context, rewind): + if settings.PROFILE: + if self.counter % 10 and self._append_cache: + append = self._append_cache + else: + self._append_cache = append = ( + ('Memory', '{0:.2f} Mb'.format(memory_usage())), + # ('Total time', '{0} s'.format(execution_time(harness))), + ) + else: + append = () + self.write(graph_context, prefix=self.prefix, append=append, rewind=rewind) + self.counter += 1 def memory_usage(): import os, psutil process = psutil.Process(os.getpid()) - return process.memory_info()[0] / float(2**20) \ No newline at end of file + return process.memory_info()[0] / float(2 ** 20) diff --git a/bonobo/ext/edgy/__init__.py b/bonobo/ext/edgy/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bonobo/ext/edgy/project/__init__.py b/bonobo/ext/edgy/project/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bonobo/ext/edgy/project/feature.py b/bonobo/ext/edgy/project/feature.py deleted file mode 100644 index 5a01f97..0000000 --- a/bonobo/ext/edgy/project/feature.py +++ /dev/null @@ -1,22 +0,0 @@ -try: - import edgy.project -except ImportError as e: - import logging - - logging.exception('You must install edgy.project to use this.') - -import os - -from edgy.project.events import subscribe -from edgy.project.feature import Feature, SUPPORT_PRIORITY - - -class BonoboFeature(Feature): - requires = {'python'} - - @subscribe('edgy.project.on_start', priority=SUPPORT_PRIORITY) - def on_start(self, event): - package_path = event.setup['name'].replace('.', os.sep) - - for file in ('example_graph'): - self.render_file(os.path.join(package_path, file + '.py'), os.path.join('tornado', file + '.py.j2')) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index bbf6ab5..1242cb2 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -18,7 +18,7 @@ __all__ = [ 'Tee', 'count', 'pprint', - 'PrettyPrint', + 'PrettyPrinter', 'noop', ] @@ -85,7 +85,8 @@ class PrettyPrinter(Configurable): return ' '.join(((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)) -pprint = Tee(_pprint) +pprint = PrettyPrinter() +pprint.__name__ = 'pprint' def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True): diff --git a/bonobo/util/packages.py b/bonobo/util/packages.py new file mode 100644 index 0000000..e4de1b6 --- /dev/null +++ b/bonobo/util/packages.py @@ -0,0 +1,8 @@ +import pkg_resources +from packaging.utils import canonicalize_name + +bonobo_packages = {} +for p in pkg_resources.working_set: + name = canonicalize_name(p.project_name) + if name.startswith('bonobo'): + bonobo_packages[name] = p From aea5ecf17e45a2008d9168426ad37ab020af2e5e Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 15:17:55 +0200 Subject: [PATCH 4/9] [cli] Adds --install/-I flag to install requirements.txt prior to run when targetting a directory, needed for docker runs. --- bonobo/commands/run.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 25fe956..0f85eb8 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -1,6 +1,8 @@ import os import runpy +import pip + import bonobo DEFAULT_SERVICES_FILENAME = '_services.py' @@ -29,7 +31,7 @@ def get_default_services(filename, services=None): return services or {} -def execute(filename, module, quiet=False, verbose=False): +def execute(filename, module, install=False, quiet=False, verbose=False): from bonobo import settings if quiet: @@ -40,7 +42,12 @@ def execute(filename, module, quiet=False, verbose=False): if filename: if os.path.isdir(filename): + if install: + requirements = os.path.join(filename, 'requirements.txt') + pip.main(['install', '-qr', requirements]) filename = os.path.join(filename, DEFAULT_GRAPH_FILENAME) + elif install: + raise RuntimeError('Cannot --install on a file (only available for dirs containing requirements.txt).') context = runpy.run_path(filename, run_name='__bonobo__') elif module: context = runpy.run_module(module, run_name='__bonobo__') @@ -68,11 +75,17 @@ def execute(filename, module, quiet=False, verbose=False): ) -def register(parser): +def register_generic_run_arguments(parser): source_group = parser.add_mutually_exclusive_group(required=True) source_group.add_argument('filename', nargs='?', type=str) source_group.add_argument('--module', '-m', type=str) + return parser + + +def register(parser): + parser = register_generic_run_arguments(parser) verbosity_group = parser.add_mutually_exclusive_group() verbosity_group.add_argument('--quiet', '-q', action='store_true') verbosity_group.add_argument('--verbose', '-v', action='store_true') + parser.add_argument('--install', '-I', action='store_true') return execute From 7e28eeddd6009e4450f0daa668b5e385a8c2d25b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 15:25:22 +0200 Subject: [PATCH 5/9] Rename packages package as it looks like it cause trouble when uploading to pypi. --- bonobo/commands/version.py | 2 +- bonobo/util/{packages.py => pkgs.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename bonobo/util/{packages.py => pkgs.py} (100%) diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index c6ac9e8..ceeec3b 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -1,5 +1,5 @@ import bonobo -from bonobo.util.packages import bonobo_packages +from bonobo.util.pkgs import bonobo_packages def format_version(mod, *, name=None, quiet=False): diff --git a/bonobo/util/packages.py b/bonobo/util/pkgs.py similarity index 100% rename from bonobo/util/packages.py rename to bonobo/util/pkgs.py From 1afd8746eb0d4c3912e9c59f98bc44a94cf2e3b9 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 16:08:10 +0200 Subject: [PATCH 6/9] [pm] Moving project artifact management to next edgy.project version. --- Makefile | 8 +- Projectfile | 93 ++++++++++-------------- bonobo/_api.py | 1 + bonobo/commands/init.py | 4 +- bonobo/commands/version.py | 4 +- bonobo/examples/files/pickle_handlers.py | 15 ++-- bonobo/examples/nodes/slow.py | 1 - bonobo/ext/console.py | 5 +- bonobo/nodes/basics.py | 4 +- requirements-dev.txt | 25 +++---- requirements-jupyter.txt | 5 +- requirements.txt | 9 ++- setup.py | 27 +++---- tests/io/test_pickle.py | 5 +- 14 files changed, 92 insertions(+), 114 deletions(-) diff --git a/Makefile b/Makefile index 3ea8912..022a1ca 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-05-22 19:54:27.969596 +# Updated at 2017-05-27 16:06:39.394642 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -20,8 +20,9 @@ SPHINX_SOURCEDIR ?= docs SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build YAPF ?= $(PYTHON_DIRNAME)/yapf YAPF_OPTIONS ?= -rip +VERSION ?= $(shell git describe 2>/dev/null || echo dev) -.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev lint test +.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test # Installs the local project dependencies. install: @@ -39,9 +40,6 @@ install-dev: clean: rm -rf build dist *.egg-info -lint: install-dev - $(PYTHON_DIRNAME)/pylint --py3k $(PACKAGE) -f html > pylint.html - test: install-dev $(PYTEST) $(PYTEST_OPTIONS) tests diff --git a/Projectfile b/Projectfile index d8fcf44..a04e297 100644 --- a/Projectfile +++ b/Projectfile @@ -1,69 +1,52 @@ # bonobo (see github.com/python-edgy/project) -name = 'bonobo' -description = 'Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.' -license = 'Apache License, Version 2.0' +from edgy.project import require -url = 'https://www.bonobo-project.org/' -download_url = 'https://github.com/python-bonobo/bonobo/tarball/{version}' +pytest = require('pytest') +python = require('python') +sphinx = require('sphinx') +yapf = require('yapf') -author = 'Romain Dorgueil' -author_email = 'romain@dorgueil.net' +python.setup( + name='bonobo', + description='Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.', + license='Apache License, Version 2.0', + url='https://www.bonobo-project.org/', + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}', + author='Romain Dorgueil', + author_email='romain@dorgueil.net', + data_files=[ + ('share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', + 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map', + ]), + ], -enable_features = { - 'make', - 'sphinx', # should install sphinx - 'pytest', # should install pytest/pytest-cov/coverage - 'git', - 'pylint', - 'python', - 'yapf', # should install yapf -} + entry_points={ + 'console_scripts': [ + 'bonobo = bonobo.commands:entrypoint', + ], + 'bonobo.commands': [ + 'init = bonobo.commands.init:register', + 'run = bonobo.commands.run:register', + 'version = bonobo.commands.version:register', + ], + } -# stricts deendencies in requirements.txt -install_requires = [ +) + +python.add_requirements( 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0', -] - -extras_require = { - 'dev': [ - 'coverage >=4,<5', - 'pylint >=1,<2', - 'pytest >=3,<4', - 'pytest-cov >=2,<3', + dev=[ 'pytest-timeout >=1,<2', - 'sphinx', - 'yapf', ], - 'jupyter': [ + jupyter=[ 'jupyter >=1.0,<1.1', - 'ipywidgets >=6.0.0.beta5' - ], -} - -data_files = [ - ('share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', - 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map', - ]), -] - -entry_points = { - 'console_scripts': [ - 'bonobo = bonobo.commands:entrypoint', - ], - 'bonobo.commands': [ - 'init = bonobo.commands.init:register', - 'run = bonobo.commands.run:register', - 'version = bonobo.commands.version:register', - ], -} - -@listen('edgy.project.feature.make.on_generate', priority=10) -def on_make_generate_docker_targets(event): - event.makefile['SPHINX_SOURCEDIR'] = 'docs' + 'ipywidgets >=6.0.0.beta5', + ] +) diff --git a/bonobo/_api.py b/bonobo/_api.py index 1eed59f..9317e31 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -6,6 +6,7 @@ from bonobo.util.objects import get_name __all__ = [] + def register_api(x, __all__=__all__): __all__.append(get_name(x)) return x diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index d13a21f..ad3c52a 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -9,7 +9,9 @@ def execute(name): 'You must install "cookiecutter" to use this command.\n\n $ pip install edgy.project\n' ) from exc - return cookiecutter('https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True) + return cookiecutter( + 'https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True + ) def register(parser): diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index ceeec3b..fda6c45 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -4,9 +4,7 @@ from bonobo.util.pkgs import bonobo_packages def format_version(mod, *, name=None, quiet=False): return ('{name} {version}' if quiet else '{name} v.{version} (in {location})').format( - name=name or mod.__name__, - version=mod.__version__, - location=bonobo_packages[name or mod.__name__].location + name=name or mod.__name__, version=mod.__version__, location=bonobo_packages[name or mod.__name__].location ) diff --git a/bonobo/examples/files/pickle_handlers.py b/bonobo/examples/files/pickle_handlers.py index c00b3fa..e6f3dcc 100644 --- a/bonobo/examples/files/pickle_handlers.py +++ b/bonobo/examples/files/pickle_handlers.py @@ -6,7 +6,9 @@ import os def cleanse_sms(row): if row['category'] == 'spam': - row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + ('...' if len(row['sms']) > 50 else '') + row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + ( + '...' if len(row['sms']) > 50 else '' + ) else: row['sms_clean'] = row['sms'] @@ -14,14 +16,13 @@ def cleanse_sms(row): graph = bonobo.Graph( - bonobo.PickleReader('spam.pkl'), # spam.pkl is within the gzipped tarball + bonobo.PickleReader('spam.pkl' + ), # spam.pkl is within the gzipped tarball cleanse_sms, print ) - if __name__ == '__main__': - ''' This example shows how a different file system service can be injected into a transformation (as compressing pickled objects often makes sense @@ -51,8 +52,10 @@ if __name__ == '__main__': ''' services = { - 'fs': TarFS( - os.path.join(bonobo.get_examples_path(), 'datasets', 'spam.tgz') + 'fs': + TarFS( + os.path. + join(bonobo.get_examples_path(), 'datasets', 'spam.tgz') ) } bonobo.run(graph, services=services) diff --git a/bonobo/examples/nodes/slow.py b/bonobo/examples/nodes/slow.py index 703ebc2..b9623af 100644 --- a/bonobo/examples/nodes/slow.py +++ b/bonobo/examples/nodes/slow.py @@ -1,7 +1,6 @@ import bonobo import time - from bonobo.constants import NOT_MODIFIED diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index 6a3ef8e..f30fae0 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -90,7 +90,8 @@ class ConsoleOutputPlugin(Plugin): ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), CLEAR_EOL ) - ), file=sys.stderr + ), + file=sys.stderr ) t_cnt += 1 @@ -116,4 +117,4 @@ class ConsoleOutputPlugin(Plugin): def memory_usage(): import os, psutil process = psutil.Process(os.getpid()) - return process.memory_info()[0] / float(2 ** 20) + return process.memory_info()[0] / float(2**20) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 1242cb2..c21757a 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -82,7 +82,9 @@ class PrettyPrinter(Configurable): return ' '.join(((' ' if i else '-'), str(item), ':', str(value).strip())) def _format_console(self, i, item, value): - return ' '.join(((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)) + return ' '.join( + ((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL) + ) pprint = PrettyPrinter() diff --git a/requirements-dev.txt b/requirements-dev.txt index 1cdb4ad..3e50f5d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,27 +1,24 @@ -e .[dev] - alabaster==0.7.10 -astroid==1.5.2 babel==2.4.0 -coverage==4.3.4 +certifi==2017.4.17 +chardet==3.0.3 +coverage==4.4.1 docutils==0.13.1 +idna==2.5 imagesize==0.7.1 -isort==4.2.5 jinja2==2.9.6 -lazy-object-proxy==1.2.2 markupsafe==1.0 -mccabe==0.6.1 py==1.4.33 pygments==2.2.0 -pylint==1.7.1 -pytest-cov==2.4.0 +pytest-cov==2.5.1 pytest-timeout==1.2.0 -pytest==3.0.7 +pytest==3.1.0 pytz==2017.2 -requests==2.13.0 +requests==2.16.1 six==1.10.0 snowballstemmer==1.2.1 -sphinx-rtd-theme==0.2.4 -sphinx==1.5.5 -wrapt==1.10.10 -yapf==0.16.1 +sphinx==1.6.1 +sphinxcontrib-websupport==1.0.1 +typing==3.6.1 +urllib3==1.21.1 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 1cf03f6..1e98481 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,5 +1,4 @@ -e .[jupyter] - appnope==0.1.0 bleach==2.0.0 decorator==4.0.11 @@ -18,7 +17,7 @@ jupyter-core==4.3.0 jupyter==1.0.0 markupsafe==1.0 mistune==0.7.4 -nbconvert==5.1.1 +nbconvert==5.2.1 nbformat==4.3.0 notebook==5.0.0 pandocfilters==1.4.1 @@ -33,7 +32,7 @@ qtconsole==4.3.0 simplegeneric==0.8.1 six==1.10.0 terminado==0.6 -testpath==0.3 +testpath==0.3.1 tornado==4.5.1 traitlets==4.3.2 wcwidth==0.1.7 diff --git a/requirements.txt b/requirements.txt index b11cfa3..86d900d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,15 @@ -e . - appdirs==1.4.3 +certifi==2017.4.17 +chardet==3.0.3 colorama==0.3.9 enum34==1.1.6 fs==2.0.3 -pbr==3.0.0 +idna==2.5 +pbr==3.0.1 psutil==5.2.2 pytz==2017.2 -requests==2.13.0 +requests==2.16.1 six==1.10.0 stevedore==1.21.0 +urllib3==1.21.1 diff --git a/setup.py b/setup.py index feabb8c..62064d1 100644 --- a/setup.py +++ b/setup.py @@ -41,41 +41,34 @@ else: version = version_ns.get('__version__', 'dev') setup( - name='bonobo', + author='Romain Dorgueil', + author_email='romain@dorgueil.net', description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'python 3.5+.'), license='Apache License, Version 2.0', - install_requires=[ - 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0' - ], + name='bonobo', version=version, long_description=long_description, classifiers=classifiers, packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, - data_files=[ - ( - 'share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map' - ] - ) + install_requires=[ + 'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'psutil (>= 5.2, < 6.0)', 'requests (>= 2.0, < 3.0)', + 'stevedore (>= 1.21, < 2.0)' ], extras_require={ 'dev': [ - 'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', - 'sphinx_rtd_theme', 'yapf' + 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)', + 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)' ], - 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] + 'jupyter': ['ipywidgets (>= 6.0.0.beta5)', 'jupyter (>= 1.0, < 1.1)'] }, entry_points={ 'bonobo.commands': [ 'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register' ], - 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], - 'edgy.project.features': ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature'] + 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] }, url='https://www.bonobo-project.org/', download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), diff --git a/tests/io/test_pickle.py b/tests/io/test_pickle.py index a8333ef..662fc4a 100644 --- a/tests/io/test_pickle.py +++ b/tests/io/test_pickle.py @@ -28,9 +28,8 @@ def test_write_pickled_dict_to_file(tmpdir): def test_read_pickled_list_from_file(tmpdir): fs, filename = open_fs(tmpdir), 'input.pkl' - fs.open(filename, 'wb').write(pickle.dumps([ - ['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar'] - ])) + fs.open(filename, + 'wb').write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])) reader = PickleReader(path=filename) From 665ddb560e10e3dddc6ef9aa8933103c1c55207b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 16:10:14 +0200 Subject: [PATCH 7/9] Fix version test --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 40a6ed5..280308d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -76,4 +76,4 @@ def test_version(runner, capsys): out, err = capsys.readouterr() out = out.strip() assert out.startswith('bonobo ') - assert out.endswith(__version__) + assert __version__ in out From d241fdd708d7974a29683afbd9ac8447c27aba69 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 16:26:14 +0200 Subject: [PATCH 8/9] [cli] bonobo version -qq now returns only version number --- bonobo/commands/version.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index fda6c45..bfa03a7 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -3,9 +3,20 @@ from bonobo.util.pkgs import bonobo_packages def format_version(mod, *, name=None, quiet=False): - return ('{name} {version}' if quiet else '{name} v.{version} (in {location})').format( - name=name or mod.__name__, version=mod.__version__, location=bonobo_packages[name or mod.__name__].location - ) + args = { + 'name': name or mod.__name__, + 'version': mod.__version__, + 'location': bonobo_packages[name or mod.__name__].location + } + + if not quiet: + return '{name} v.{version} (in {location})'.format(**args) + if quiet < 2: + return '{name} {version}'.format(**args) + if quiet < 3: + return '{version}'.format(**args) + + raise RuntimeError('Hard to be so quiet...') def execute(all=False, quiet=False): @@ -25,5 +36,5 @@ def execute(all=False, quiet=False): def register(parser): parser.add_argument('--all', '-a', action='store_true') - parser.add_argument('--quiet', '-q', action='store_true') + parser.add_argument('--quiet', '-q', action='count') return execute From 3a5ebff43532983aa801b377556719745e325872 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 27 May 2017 17:06:18 +0200 Subject: [PATCH 9/9] Adds packaging requirement. --- Makefile | 2 +- Projectfile | 1 + requirements-dev.txt | 2 +- requirements.txt | 4 +++- setup.py | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 022a1ca..e0ab813 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-05-27 16:06:39.394642 +# Updated at 2017-05-27 17:05:44.723397 PACKAGE ?= bonobo PYTHON ?= $(shell which python) diff --git a/Projectfile b/Projectfile index a04e297..94c8a4f 100644 --- a/Projectfile +++ b/Projectfile @@ -39,6 +39,7 @@ python.setup( python.add_requirements( 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', + 'packaging >=16,<17', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0', diff --git a/requirements-dev.txt b/requirements-dev.txt index 3e50f5d..83f50fa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,7 +15,7 @@ pytest-cov==2.5.1 pytest-timeout==1.2.0 pytest==3.1.0 pytz==2017.2 -requests==2.16.1 +requests==2.16.2 six==1.10.0 snowballstemmer==1.2.1 sphinx==1.6.1 diff --git a/requirements.txt b/requirements.txt index 86d900d..184e757 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,10 +6,12 @@ colorama==0.3.9 enum34==1.1.6 fs==2.0.3 idna==2.5 +packaging==16.8 pbr==3.0.1 psutil==5.2.2 +pyparsing==2.2.0 pytz==2017.2 -requests==2.16.1 +requests==2.16.2 six==1.10.0 stevedore==1.21.0 urllib3==1.21.1 diff --git a/setup.py b/setup.py index 62064d1..cc20417 100644 --- a/setup.py +++ b/setup.py @@ -53,8 +53,8 @@ setup( packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, install_requires=[ - 'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'psutil (>= 5.2, < 6.0)', 'requests (>= 2.0, < 3.0)', - 'stevedore (>= 1.21, < 2.0)' + 'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)', + 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)' ], extras_require={ 'dev': [