paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[pm] Moving project artifact management to next edgy.project version.

Browse Source
This commit is contained in:
Romain Dorgueil
2017-05-27 16:08:10 +02:00
parent 93cc1648a4
commit 1afd8746eb
14 changed files with 92 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -1,7 +1,7 @@
# This file has been auto-generated.
# All changes will be lost, see Projectfile.
#
# Updated at 2017-05-22 19:54:27.969596
# Updated at 2017-05-27 16:06:39.394642
PACKAGE ?= bonobo
PYTHON ?= $(shell which python)
@ -20,8 +20,9 @@ SPHINX_SOURCEDIR ?= docs
SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
YAPF ?= $(PYTHON_DIRNAME)/yapf
YAPF_OPTIONS ?= -rip
VERSION ?= $(shell git describe 2>/dev/null || echo dev)
.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev lint test
.PHONY: $(SPHINX_SOURCEDIR) clean format install install-dev test
# Installs the local project dependencies.
install:
@ -39,9 +40,6 @@ install-dev:
clean:
rm -rf build dist *.egg-info
lint: install-dev
$(PYTHON_DIRNAME)/pylint --py3k $(PACKAGE) -f html > pylint.html
test: install-dev
$(PYTEST) $(PYTEST_OPTIONS) tests

93
Projectfile
View File

@ -1,69 +1,52 @@
# bonobo (see github.com/python-edgy/project)
name = 'bonobo'
description = 'Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.'
license = 'Apache License, Version 2.0'
from edgy.project import require
url = 'https://www.bonobo-project.org/'
download_url = 'https://github.com/python-bonobo/bonobo/tarball/{version}'
pytest = require('pytest')
python = require('python')
sphinx = require('sphinx')
yapf = require('yapf')
author = 'Romain Dorgueil'
author_email = 'romain@dorgueil.net'
python.setup(
name='bonobo',
description='Bonobo, a simple, modern and atomic extract-transform-load toolkit for python 3.5+.',
license='Apache License, Version 2.0',
url='https://www.bonobo-project.org/',
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}',
author='Romain Dorgueil',
author_email='romain@dorgueil.net',
data_files=[
('share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/ext/jupyter/static/extension.js',
'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map',
]),
],
enable_features = {
'make',
'sphinx', # should install sphinx
'pytest', # should install pytest/pytest-cov/coverage
'git',
'pylint',
'python',
'yapf', # should install yapf
}
entry_points={
'console_scripts': [
'bonobo = bonobo.commands:entrypoint',
],
'bonobo.commands': [
'init = bonobo.commands.init:register',
'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register',
],
}
# stricts deendencies in requirements.txt
install_requires = [
)
python.add_requirements(
'colorama >=0.3,<1.0',
'fs >=2.0,<3.0',
'psutil >=5.2,<6.0',
'requests >=2.0,<3.0',
'stevedore >=1.21,<2.0',
]
extras_require = {
'dev': [
'coverage >=4,<5',
'pylint >=1,<2',
'pytest >=3,<4',
'pytest-cov >=2,<3',
dev=[
'pytest-timeout >=1,<2',
'sphinx',
'yapf',
],
'jupyter': [
jupyter=[
'jupyter >=1.0,<1.1',
'ipywidgets >=6.0.0.beta5'
],
}
data_files = [
('share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/ext/jupyter/static/extension.js',
'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map',
]),
]
entry_points = {
'console_scripts': [
'bonobo = bonobo.commands:entrypoint',
],
'bonobo.commands': [
'init = bonobo.commands.init:register',
'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register',
],
}
@listen('edgy.project.feature.make.on_generate', priority=10)
def on_make_generate_docker_targets(event):
event.makefile['SPHINX_SOURCEDIR'] = 'docs'
'ipywidgets >=6.0.0.beta5',
]
)

1
bonobo/_api.py
View File

@ -6,6 +6,7 @@ from bonobo.util.objects import get_name
__all__ = []
def register_api(x, __all__=__all__):
__all__.append(get_name(x))
return x

4
bonobo/commands/init.py
View File

@ -9,7 +9,9 @@ def execute(name):
'You must install "cookiecutter" to use this command.\n\n $ pip install edgy.project\n'
) from exc
return cookiecutter('https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True)
return cookiecutter(
'https://github.com/python-bonobo/cookiecutter-bonobo.git', extra_context={'name': name}, no_input=True
)
def register(parser):

4
bonobo/commands/version.py
View File

@ -4,9 +4,7 @@ from bonobo.util.pkgs import bonobo_packages
def format_version(mod, *, name=None, quiet=False):
return ('{name} {version}' if quiet else '{name} v.{version} (in {location})').format(
name=name or mod.__name__,
version=mod.__version__,
location=bonobo_packages[name or mod.__name__].location
name=name or mod.__name__, version=mod.__version__, location=bonobo_packages[name or mod.__name__].location
)

15
bonobo/examples/files/pickle_handlers.py
View File

@ -6,7 +6,9 @@ import os
def cleanse_sms(row):
if row['category'] == 'spam':
row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + ('...' if len(row['sms']) > 50 else '')
row['sms_clean'] = '**MARKED AS SPAM** ' + row['sms'][0:50] + (
'...' if len(row['sms']) > 50 else ''
)
else:
row['sms_clean'] = row['sms']
@ -14,14 +16,13 @@ def cleanse_sms(row):
graph = bonobo.Graph(
bonobo.PickleReader('spam.pkl'), # spam.pkl is within the gzipped tarball
bonobo.PickleReader('spam.pkl'
), # spam.pkl is within the gzipped tarball
cleanse_sms,
print
)
if __name__ == '__main__':
'''
This example shows how a different file system service can be injected
into a transformation (as compressing pickled objects often makes sense
@ -51,8 +52,10 @@ if __name__ == '__main__':
'''
services = {
'fs': TarFS(
os.path.join(bonobo.get_examples_path(), 'datasets', 'spam.tgz')
'fs':
TarFS(
os.path.
join(bonobo.get_examples_path(), 'datasets', 'spam.tgz')
)
}
bonobo.run(graph, services=services)

1
bonobo/examples/nodes/slow.py
View File

@ -1,7 +1,6 @@
import bonobo
import time
from bonobo.constants import NOT_MODIFIED

5
bonobo/ext/console.py
View File

@ -90,7 +90,8 @@ class ConsoleOutputPlugin(Plugin):
' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v)
for k, v in append), CLEAR_EOL
)
), file=sys.stderr
),
file=sys.stderr
)
t_cnt += 1
@ -116,4 +117,4 @@ class ConsoleOutputPlugin(Plugin):
def memory_usage():
import os, psutil
process = psutil.Process(os.getpid())
return process.memory_info()[0] / float(2 ** 20)
return process.memory_info()[0] / float(2**20)

4
bonobo/nodes/basics.py
View File

@ -82,7 +82,9 @@ class PrettyPrinter(Configurable):
return ' '.join(((' ' if i else '-'), str(item), ':', str(value).strip()))
def _format_console(self, i, item, value):
return ' '.join(((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL))
return ' '.join(
((' ' if i else '•'), str(item), '=', str(value).strip().replace('\n', '\n' + CLEAR_EOL), CLEAR_EOL)
)
pprint = PrettyPrinter()

25
requirements-dev.txt
View File

@ -1,27 +1,24 @@
-e .[dev]
alabaster==0.7.10
astroid==1.5.2
babel==2.4.0
coverage==4.3.4
certifi==2017.4.17
chardet==3.0.3
coverage==4.4.1
docutils==0.13.1
idna==2.5
imagesize==0.7.1
isort==4.2.5
jinja2==2.9.6
lazy-object-proxy==1.2.2
markupsafe==1.0
mccabe==0.6.1
py==1.4.33
pygments==2.2.0
pylint==1.7.1
pytest-cov==2.4.0
pytest-cov==2.5.1
pytest-timeout==1.2.0
pytest==3.0.7
pytest==3.1.0
pytz==2017.2
requests==2.13.0
requests==2.16.1
six==1.10.0
snowballstemmer==1.2.1
sphinx-rtd-theme==0.2.4
sphinx==1.5.5
wrapt==1.10.10
yapf==0.16.1
sphinx==1.6.1
sphinxcontrib-websupport==1.0.1
typing==3.6.1
urllib3==1.21.1

5
requirements-jupyter.txt
View File

@ -1,5 +1,4 @@
-e .[jupyter]
appnope==0.1.0
bleach==2.0.0
decorator==4.0.11
@ -18,7 +17,7 @@ jupyter-core==4.3.0
jupyter==1.0.0
markupsafe==1.0
mistune==0.7.4
nbconvert==5.1.1
nbconvert==5.2.1
nbformat==4.3.0
notebook==5.0.0
pandocfilters==1.4.1
@ -33,7 +32,7 @@ qtconsole==4.3.0
simplegeneric==0.8.1
six==1.10.0
terminado==0.6
testpath==0.3
testpath==0.3.1
tornado==4.5.1
traitlets==4.3.2
wcwidth==0.1.7

9
requirements.txt
View File

@ -1,12 +1,15 @@
-e .
appdirs==1.4.3
certifi==2017.4.17
chardet==3.0.3
colorama==0.3.9
enum34==1.1.6
fs==2.0.3
pbr==3.0.0
idna==2.5
pbr==3.0.1
psutil==5.2.2
pytz==2017.2
requests==2.13.0
requests==2.16.1
six==1.10.0
stevedore==1.21.0
urllib3==1.21.1

27
setup.py
View File

@ -41,41 +41,34 @@ else:
version = version_ns.get('__version__', 'dev')
setup(
name='bonobo',
author='Romain Dorgueil',
author_email='romain@dorgueil.net',
description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
'python 3.5+.'),
license='Apache License, Version 2.0',
install_requires=[
'colorama >=0.3,<1.0', 'fs >=2.0,<3.0', 'psutil >=5.2,<6.0', 'requests >=2.0,<3.0', 'stevedore >=1.21,<2.0'
],
name='bonobo',
version=version,
long_description=long_description,
classifiers=classifiers,
packages=find_packages(exclude=['ez_setup', 'example', 'test']),
include_package_data=True,
data_files=[
(
'share/jupyter/nbextensions/bonobo-jupyter', [
'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js',
'bonobo/ext/jupyter/static/index.js.map'
]
)
install_requires=[
'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'psutil (>= 5.2, < 6.0)', 'requests (>= 2.0, < 3.0)',
'stevedore (>= 1.21, < 2.0)'
],
extras_require={
'dev': [
'coverage >=4,<5', 'pylint >=1,<2', 'pytest >=3,<4', 'pytest-cov >=2,<3', 'pytest-timeout >=1,<2', 'sphinx',
'sphinx_rtd_theme', 'yapf'
'coverage (>= 4.4, < 5.0)', 'pytest (>= 3.1, < 4.0)', 'pytest-cov (>= 2.5, < 3.0)',
'pytest-timeout (>= 1, < 2)', 'sphinx (>= 1.6, < 2.0)'
],
'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5']
'jupyter': ['ipywidgets (>= 6.0.0.beta5)', 'jupyter (>= 1.0, < 1.1)']
},
entry_points={
'bonobo.commands': [
'init = bonobo.commands.init:register', 'run = bonobo.commands.run:register',
'version = bonobo.commands.version:register'
],
'console_scripts': ['bonobo = bonobo.commands:entrypoint'],
'edgy.project.features': ['bonobo = '
'bonobo.ext.edgy.project.feature:BonoboFeature']
'console_scripts': ['bonobo = bonobo.commands:entrypoint']
},
url='https://www.bonobo-project.org/',
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version),

5
tests/io/test_pickle.py
View File

@ -28,9 +28,8 @@ def test_write_pickled_dict_to_file(tmpdir):
def test_read_pickled_list_from_file(tmpdir):
fs, filename = open_fs(tmpdir), 'input.pkl'
fs.open(filename, 'wb').write(pickle.dumps([
['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']
]))
fs.open(filename,
'wb').write(pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']]))
reader = PickleReader(path=filename)