diff --git a/Makefile b/Makefile index 3ea8912..022a1ca 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # This file has been auto-generated. # All changes will be lost, see Projectfile. # -# Updated at 2017-05-22 19:54:27.969596 +# Updated at 2017-05-27 16:06:39.394642 PACKAGE ?= bonobo PYTHON ?= $(shell which python) @@ -20,8 +20,9 @@ SPHINX_SOURCEDIR ?= docs SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build YAPF ?= $(PYTHON_DIRNAME)/yapf YAPF_OPTIONS ?= -rip +VERSION ?= $(shell git describe 2>/dev/null || echo dev) -.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev lint test +.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test # Installs the local project dependencies. install: @@ -39,9 +40,6 @@ install-dev: clean: rm -rf build dist *.egg-info -lint: install-dev - $(PYTHON_DIRNAME)/pylint --py3k $(PACKAGE) -f html > pylint.html - test: install-dev $(PYTEST) $(PYTEST_OPTIONS) tests diff --git a/Projectfile b/Projectfile index d8fcf44..a04e297 100644 --- a/Projectfile +++ b/Projectfile @@ -1,69 +1,52 @@ # bonobo (see github.com/python-edgy/project) -name = 'bonobo' -description = 'Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.' -license = 'Apache License, Version 2.0' +from edgy.project import require -url = 'https://www.bonobo-project.org/' -download_url = 'https://github.com/python-bonobo/bonobo/tarball/{version}' +pytest = require('pytest') +python = require('python') +sphinx = require('sphinx') +yapf = require('yapf') -author = 'Romain Dorgueil' -author_email = 'romain@dorgueil.net' +python.setup( + name='bonobo', + description='Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.', + license='Apache License, Version 2.0', + url='https://www.bonobo-project.org/', + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}', + author='Romain Dorgueil', + author_email='romain@dorgueil.net', + data_files=[ + ('share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', + 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map', + ]), + ], -enable_features = { - 'make', - 'sphinx', # should install sphinx - 'pytest', # should install pytest/pytest-cov/coverage - 'git', - 'pylint', - 'python', - 'yapf', # should install yapf -} + entry_points={ + 'console_scripts': [ + 'bonobo = bonobo.commands:entrypoint', + ], + 'bonobo.commands': [ + 'init = bonobo.commands.init:register', + 'run = bonobo.commands.run:register', + 'version = bonobo.commands.version:register', + ], + } -# stricts deendencies in requirements.txt -install_requires = [ +) + +python.add_requirements( 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0', -] - -extras_require = { - 'dev': [ - 'coverage >=4,<5', - 'pylint >=1,<2', - 'pytest >=3,<4', - 'pytest-cov >=2,<3', + dev=[ 'pytest-timeout >=1,<2', - 'sphinx', - 'yapf', ], - 'jupyter': [ + jupyter=[ 'jupyter >=1.0,<1.1', - 'ipywidgets >=6.0.0.beta5' - ], -} - -data_files = [ - ('share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', - 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map', - ]), -] - -entry_points = { - 'console_scripts': [ - 'bonobo = bonobo.commands:entrypoint', - ], - 'bonobo.commands': [ - 'init = bonobo.commands.init:register', - 'run = bonobo.commands.run:register', - 'version = bonobo.commands.version:register', - ], -} - -@listen('edgy.project.feature.make.on_generate', priority=10) -def on_make_generate_docker_targets(event): - event.makefile['SPHINX_SOURCEDIR'] = 'docs' + 'ipywidgets >=6.0.0.beta5', + ] +) diff --git a/bonobo/_api.py b/bonobo/_api.py index 1eed59f..9317e31 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -6,6 +6,7 @@ from bonobo.util.objects import get_name __all__ = [] + def register_api(x, __all__=__all__): __all__.append(get_name(x)) return x diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index d13a21f..ad3c52a 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -9,7 +9,9 @@ def execute(name): 'You must install "cookiecutter" to use this command.\n\n $ pip install edgy.project\n' ) from exc - return cookiecutter('https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True) + return cookiecutter( + 'https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True + ) def register(parser): diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index ceeec3b..fda6c45 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -4,9 +4,7 @@ from bonobo.util.pkgs import bonobo_packages def format_version(mod, *, name=None, quiet=False): return ('{name} {version}' if quiet else '{name} v.{version} (in {location})').format( - name=name or mod.__name__, - version=mod.__version__, - location=bonobo_packages[name or mod.__name__].location + name=name or mod.__name__, version=mod.__version__, location=bonobo_packages[name or mod.__name__].location ) diff --git a/bonobo/examples/files/pickle_handlers.py b/bonobo/examples/files/pickle_handlers.py index c00b3fa..e6f3dcc 100644 --- a/bonobo/examples/files/pickle_handlers.py +++ b/bonobo/examples/files/pickle_handlers.py @@ -6,7 +6,9 @@ import os def cleanse_sms(row): if row['category'] == 'spam': - row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + ('...' if len(row['sms']) > 50 else '') + row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + ( + '...' if len(row['sms']) > 50 else '' + ) else: row['sms_clean'] = row['sms'] @@ -14,14 +16,13 @@ def cleanse_sms(row): graph = bonobo.Graph( - bonobo.PickleReader('spam.pkl'), # spam.pkl is within the gzipped tarball + bonobo.PickleReader('spam.pkl' + ), # spam.pkl is within the gzipped tarball cleanse_sms, print ) - if __name__ == '__main__': - ''' This example shows how a different file system service can be injected into a transformation (as compressing pickled objects often makes sense @@ -51,8 +52,10 @@ if __name__ == '__main__': ''' services = { - 'fs': TarFS( - os.path.join(bonobo.get_examples_path(), 'datasets', 'spam.tgz') + 'fs': + TarFS( + os.path. + join(bonobo.get_examples_path(), 'datasets', 'spam.tgz') ) } bonobo.run(graph, services=services) diff --git a/bonobo/examples/nodes/slow.py b/bonobo/examples/nodes/slow.py index 703ebc2..b9623af 100644 --- a/bonobo/examples/nodes/slow.py +++ b/bonobo/examples/nodes/slow.py @@ -1,7 +1,6 @@ import bonobo import time - from bonobo.constants import NOT_MODIFIED diff --git a/bonobo/ext/console.py b/bonobo/ext/console.py index 6a3ef8e..f30fae0 100644 --- a/bonobo/ext/console.py +++ b/bonobo/ext/console.py @@ -90,7 +90,8 @@ class ConsoleOutputPlugin(Plugin): ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), CLEAR_EOL ) - ), file=sys.stderr + ), + file=sys.stderr ) t_cnt += 1 @@ -116,4 +117,4 @@ class ConsoleOutputPlugin(Plugin): def memory_usage(): import os, psutil process = psutil.Process(os.getpid()) - return process.memory_info()[0] / float(2 ** 20) + return process.memory_info()[0] / float(2**20) diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 1242cb2..c21757a 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -82,7 +82,9 @@ class PrettyPrinter(Configurable): return ' '.join(((' ' if i else '-'), str(item), ':', str(value).strip())) def _format_console(self, i, item, value): - return ' '.join(((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)) + return ' '.join( + ((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL) + ) pprint = PrettyPrinter() diff --git a/requirements-dev.txt b/requirements-dev.txt index 1cdb4ad..3e50f5d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,27 +1,24 @@ -e .[dev] - alabaster==0.7.10 -astroid==1.5.2 babel==2.4.0 -coverage==4.3.4 +certifi==2017.4.17 +chardet==3.0.3 +coverage==4.4.1 docutils==0.13.1 +idna==2.5 imagesize==0.7.1 -isort==4.2.5 jinja2==2.9.6 -lazy-object-proxy==1.2.2 markupsafe==1.0 -mccabe==0.6.1 py==1.4.33 pygments==2.2.0 -pylint==1.7.1 -pytest-cov==2.4.0 +pytest-cov==2.5.1 pytest-timeout==1.2.0 -pytest==3.0.7 +pytest==3.1.0 pytz==2017.2 -requests==2.13.0 +requests==2.16.1 six==1.10.0 snowballstemmer==1.2.1 -sphinx-rtd-theme==0.2.4 -sphinx==1.5.5 -wrapt==1.10.10 -yapf==0.16.1 +sphinx==1.6.1 +sphinxcontrib-websupport==1.0.1 +typing==3.6.1 +urllib3==1.21.1 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 1cf03f6..1e98481 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -1,5 +1,4 @@ -e .[jupyter] - appnope==0.1.0 bleach==2.0.0 decorator==4.0.11 @@ -18,7 +17,7 @@ jupyter-core==4.3.0 jupyter==1.0.0 markupsafe==1.0 mistune==0.7.4 -nbconvert==5.1.1 +nbconvert==5.2.1 nbformat==4.3.0 notebook==5.0.0 pandocfilters==1.4.1 @@ -33,7 +32,7 @@ qtconsole==4.3.0 simplegeneric==0.8.1 six==1.10.0 terminado==0.6 -testpath==0.3 +testpath==0.3.1 tornado==4.5.1 traitlets==4.3.2 wcwidth==0.1.7 diff --git a/requirements.txt b/requirements.txt index b11cfa3..86d900d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,15 @@ -e . - appdirs==1.4.3 +certifi==2017.4.17 +chardet==3.0.3 colorama==0.3.9 enum34==1.1.6 fs==2.0.3 -pbr==3.0.0 +idna==2.5 +pbr==3.0.1 psutil==5.2.2 pytz==2017.2 -requests==2.13.0 +requests==2.16.1 six==1.10.0 stevedore==1.21.0 +urllib3==1.21.1 diff --git a/setup.py b/setup.py index feabb8c..62064d1 100644 --- a/setup.py +++ b/setup.py @@ -41,41 +41,34 @@ else: version = version_ns.get('__version__', 'dev') setup( - name='bonobo', + author='Romain Dorgueil', + author_email='romain@dorgueil.net', description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'python 3.5+.'), license='Apache License, Version 2.0', - install_requires=[ - 'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0' - ], + name='bonobo', version=version, long_description=long_description, classifiers=classifiers, packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, - data_files=[ - ( - 'share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map' - ] - ) + install_requires=[ + 'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'psutil (>= 5.2, < 6.0)', 'requests (>= 2.0, < 3.0)', + 'stevedore (>= 1.21, < 2.0)' ], extras_require={ 'dev': [ - 'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx', - 'sphinx_rtd_theme', 'yapf' + 'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)', + 'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)' ], - 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] + 'jupyter': ['ipywidgets (>= 6.0.0.beta5)', 'jupyter (>= 1.0, < 1.1)'] }, entry_points={ 'bonobo.commands': [ 'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register' ], - 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], - 'edgy.project.features': ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature'] + 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] }, url='https://www.bonobo-project.org/', download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), diff --git a/tests/io/test_pickle.py b/tests/io/test_pickle.py index a8333ef..662fc4a 100644 --- a/tests/io/test_pickle.py +++ b/tests/io/test_pickle.py @@ -28,9 +28,8 @@ def test_write_pickled_dict_to_file(tmpdir): def test_read_pickled_list_from_file(tmpdir): fs, filename = open_fs(tmpdir), 'input.pkl' - fs.open(filename, 'wb').write(pickle.dumps([ - ['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar'] - ])) + fs.open(filename, + 'wb').write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']])) reader = PickleReader(path=filename)