paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #311 from hartym/backport/kasei-exclusive-locks

Browse Source 
Backport of MR from Kasei : exclusive locks
This commit is contained in:
Romain Dorgueil
2019-05-16 15:07:48 +02:00
committed by GitHub
parent 0659263224 2e40f84045
commit 62a6a5211f
18 changed files with 174 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Makefile
View File

@ -1,7 +1,8 @@
# Generated by Medikit 0.6.3 on 2018-08-11.
# Generated by Medikit 0.7.1 on 2019-05-16.
# All changes will be overriden.
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
PACKAGE ?= bonobo
PYTHON ?= $(shell which python || echo python)
PYTHON_BASENAME ?= $(shell basename $(PYTHON))
@ -29,7 +30,7 @@ SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build
SPHINX_AUTOBUILD ?= $(PYTHON_DIRNAME)/sphinx-autobuild
MEDIKIT ?= $(PYTHON) -m medikit
MEDIKIT_UPDATE_OPTIONS ?=
MEDIKIT_VERSION ?= 0.6.3
MEDIKIT_VERSION ?= 0.7.1
.PHONY: $(SPHINX_SOURCEDIR) clean format help install install-dev install-docker install-jupyter install-sqlalchemy medikit quick test update update-requirements watch-$(SPHINX_SOURCEDIR)
@ -135,5 +136,5 @@ update-requirements: ## Update project artifacts using medikit, including requ
help: ## Shows available commands.
@echo "Available commands:"
@echo
@grep -E '^[a-zA-Z_-]+:.*?##[\s]?.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; {printf " make \033[36m%-30s\033[0m %s\n", $$1, $$2}'
@grep -E '^[a-zA-Z_-]+:.*?##[\s]?.*$$' --no-filename $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; {printf " make \033[36m%-30s\033[0m %s\n", $$1, $$2}'
@echo

2
Projectfile
View File

@ -45,7 +45,7 @@ python.add_requirements(
'graphviz >=0.8,<0.9',
'jinja2 ~=2.9',
'mondrian ~=0.7',
'packaging ~=17.0',
'packaging ~=19.0',
'psutil ~=5.4',
'python-slugify ~=1.2.0',
'requests ~=2.0',

8
bonobo/__init__.py
View File

File diff suppressed because one or more lines are too long

1
bonobo/commands/__init__.py
View File

@ -2,6 +2,7 @@ import argparse
import logging
import mondrian
from bonobo import settings
from bonobo.commands.base import BaseCommand, BaseGraphCommand

4
bonobo/config/__init__.py
View File

@ -7,8 +7,8 @@ configurable transformations, either class-based or function-based.
from bonobo.config.configurables import Configurable
from bonobo.config.functools import transformation_factory
from bonobo.config.options import Method, Option
from bonobo.config.processors import ContextProcessor, use_context, use_context_processor, use_raw_input, use_no_input
from bonobo.config.services import Container, Exclusive, Service, use, create_container
from bonobo.config.processors import ContextProcessor, use_context, use_context_processor, use_no_input, use_raw_input
from bonobo.config.services import Container, Exclusive, Service, create_container, use
from bonobo.util import deprecated_alias
requires = deprecated_alias('requires', use)

2
bonobo/config/services.py
View File

@ -146,12 +146,14 @@ class Exclusive(ContextDecorator):
"""
_locks = {}
_locks_creation_lock = threading.Lock()
def __init__(self, wrapped):
self._wrapped = wrapped
def get_lock(self):
_id = id(self._wrapped)
with Exclusive._locks_creation_lock:
if not _id in Exclusive._locks:
Exclusive._locks[_id] = threading.RLock()
return Exclusive._locks[_id]

2
bonobo/contrib/django/__init__.py
View File

@ -6,7 +6,7 @@ This module contains all tools for Bonobo and Django to interract nicely.
"""
from .utils import create_or_update
from .commands import ETLCommand
from .utils import create_or_update
__all__ = ['ETLCommand', 'create_or_update']

4
bonobo/contrib/django/commands.py
View File

@ -2,13 +2,13 @@ from logging import getLogger
from types import GeneratorType
from colorama import Back, Fore, Style
from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from mondrian import term
import bonobo
from bonobo.plugins.console import ConsoleOutputPlugin
from bonobo.util.term import CLEAR_EOL
from django.core.management import BaseCommand
from django.core.management.base import OutputWrapper
from .utils import create_or_update

7
bonobo/contrib/google/__init__.py
View File

@ -1,14 +1,15 @@
import os
# https://developers.google.com/api-client-library/python/guide/aaa_oauth
# pip install google-api-python-client (1.6.4)
import httplib2
from apiclient import discovery
from oauth2client import client, tools
from oauth2client.file import Storage
from oauth2client.tools import argparser
# https://developers.google.com/api-client-library/python/guide/aaa_oauth
# pip install google-api-python-client (1.6.4)
HOME_DIR = os.path.expanduser('~')
GOOGLE_SECRETS = os.path.join(HOME_DIR, '.cache/secrets/client_secrets.json')

2
bonobo/contrib/opendatasoft/__init__.py
View File

@ -3,8 +3,8 @@ from urllib.parse import urlencode
import requests # todo: make this a service so we can substitute it ?
from bonobo.config import Option
from bonobo.config.processors import ContextProcessor
from bonobo.config.configurables import Configurable
from bonobo.config.processors import ContextProcessor
from bonobo.util.objects import ValueHolder

12
bonobo/util/__init__.py
View File

@ -6,17 +6,9 @@ and inspect transformations, graphs, and nodes.
from bonobo.util.collections import cast, ensure_tuple, sortedlist, tuplize
from bonobo.util.compat import deprecated, deprecated_alias
from bonobo.util.inspect import (
inspect_node,
isconfigurable,
isconfigurabletype,
iscontextprocessor,
isdict,
ismethod,
isoption,
istuple,
istype,
inspect_node, isconfigurable, isconfigurabletype, iscontextprocessor, isdict, ismethod, isoption, istuple, istype
)
from bonobo.util.objects import get_name, get_attribute_or_create, ValueHolder
from bonobo.util.objects import ValueHolder, get_attribute_or_create, get_name
# Bonobo's util API
__all__ = [

6
bonobo/util/api.py
View File

@ -18,9 +18,9 @@ class ApiHelper:
assert (
len(parameters) > 0 and parameters[0] == 'graph'
), 'First parameter of a graph api function must be "graph".'
assert required_parameters.intersection(parameters) == required_parameters, (
'Graph api functions must define the following parameters: ' + ', '.join(sorted(required_parameters))
)
assert (
required_parameters.intersection(parameters) == required_parameters
), 'Graph api functions must define the following parameters: ' + ', '.join(sorted(required_parameters))
self.__all__.append(get_name(x))
return x

59
requirements-dev.txt
View File

@ -1,41 +1,40 @@
-e .[dev]
-r requirements.txt
alabaster==0.7.11
arrow==0.12.1
atomicwrites==1.1.5
attrs==18.1.0
alabaster==0.7.12
arrow==0.13.1
atomicwrites==1.3.0
attrs==19.1.0
babel==2.6.0
binaryornot==0.4.4
certifi==2018.4.16
certifi==2019.3.9
chardet==3.0.4
click==6.7
click==7.0
cookiecutter==1.5.1
coverage==4.5.1
coverage==4.5.3
docutils==0.14
future==0.16.0
idna==2.7
imagesize==1.0.0
future==0.17.1
idna==2.8
imagesize==1.1.0
jinja2-time==0.2.0
jinja2==2.10
markupsafe==1.0
more-itertools==4.3.0
packaging==17.1
pathlib2==2.3.2
pluggy==0.7.1
poyo==0.4.1
py==1.5.4
pygments==2.2.0
pyparsing==2.2.0
pytest-cov==2.5.1
pytest-timeout==1.3.1
pytest==3.7.1
python-dateutil==2.7.3
pytz==2018.5
requests==2.19.1
six==1.11.0
jinja2==2.10.1
markupsafe==1.1.1
more-itertools==7.0.0
packaging==19.0
pluggy==0.11.0
poyo==0.4.2
py==1.8.0
pygments==2.4.0
pyparsing==2.4.0
pytest-cov==2.7.1
pytest-timeout==1.3.3
pytest==3.10.1
python-dateutil==2.8.0
pytz==2019.1
requests==2.21.0
six==1.12.0
snowballstemmer==1.2.1
sphinx-sitemap==0.2
sphinx==1.7.6
sphinx==1.8.5
sphinxcontrib-websupport==1.1.0
urllib3==1.23
whichcraft==0.4.1
urllib3==1.24.3
whichcraft==0.5.2

36
requirements-docker.txt
View File

@ -2,29 +2,29 @@
-r requirements.txt
appdirs==1.4.3
bonobo-docker==0.6.0
certifi==2018.4.16
cached-property==1.5.1
certifi==2019.3.9
chardet==3.0.4
colorama==0.3.9
docker-pycreds==0.3.0
docker-pycreds==0.4.0
docker==2.7.0
fs==2.0.27
fs==2.4.5
graphviz==0.8.4
idna==2.7
jinja2==2.10
markupsafe==1.0
idna==2.8
jinja2==2.10.1
markupsafe==1.1.1
mondrian==0.8.0
packaging==17.1
pbr==4.2.0
psutil==5.4.6
pyparsing==2.2.0
python-slugify==1.2.5
pytz==2018.5
requests==2.19.1
pbr==5.2.0
psutil==5.6.2
pyparsing==2.4.0
python-slugify==1.2.6
pytz==2019.1
requests==2.21.0
semantic-version==2.6.0
six==1.11.0
stevedore==1.29.0
typing==3.6.4
unidecode==1.0.22
urllib3==1.23
websocket-client==0.48.0
six==1.12.0
stevedore==1.30.1
unidecode==1.0.23
urllib3==1.24.3
websocket-client==0.56.0
whistle==1.0.1

59
requirements-jupyter.txt
View File

@ -1,44 +1,45 @@
-e .[jupyter]
-r requirements.txt
appnope==0.1.0
attrs==19.1.0
backcall==0.1.0
bleach==2.1.3
decorator==4.3.0
entrypoints==0.2.3
html5lib==1.0.1
ipykernel==4.8.2
bleach==3.1.0
decorator==4.4.0
defusedxml==0.6.0
entrypoints==0.3
ipykernel==5.1.1
ipython-genutils==0.2.0
ipython==6.5.0
ipython==7.5.0
ipywidgets==6.0.1
jedi==0.12.1
jinja2==2.10
jsonschema==2.6.0
jupyter-client==5.2.3
jupyter-console==5.2.0
jedi==0.13.3
jinja2==2.10.1
jsonschema==3.0.1
jupyter-client==5.2.4
jupyter-console==6.0.0
jupyter-core==4.4.0
jupyter==1.0.0
markupsafe==1.0
mistune==0.8.3
nbconvert==5.3.1
markupsafe==1.1.1
mistune==0.8.4
nbconvert==5.5.0
nbformat==4.4.0
notebook==5.6.0
notebook==5.7.8
pandocfilters==1.4.2
parso==0.3.1
pexpect==4.6.0
pickleshare==0.7.4
prometheus-client==0.3.1
prompt-toolkit==1.0.15
parso==0.4.0
pexpect==4.7.0
pickleshare==0.7.5
prometheus-client==0.6.0
prompt-toolkit==2.0.9
ptyprocess==0.6.0
pygments==2.2.0
python-dateutil==2.7.3
pyzmq==17.1.2
qtconsole==4.3.1
pygments==2.4.0
pyrsistent==0.15.2
python-dateutil==2.8.0
pyzmq==18.0.1
qtconsole==4.4.4
send2trash==1.5.0
simplegeneric==0.8.1
six==1.11.0
terminado==0.8.1
testpath==0.3.1
tornado==5.1
six==1.12.0
terminado==0.8.2
testpath==0.4.2
tornado==6.0.2
traitlets==4.3.2
wcwidth==0.1.7
webencodings==0.5.1

34
requirements-sqlalchemy.txt
View File

@ -2,26 +2,26 @@
-r requirements.txt
appdirs==1.4.3
bonobo-sqlalchemy==0.6.0
certifi==2018.4.16
cached-property==1.5.1
certifi==2019.3.9
chardet==3.0.4
colorama==0.3.9
fs==2.0.27
fs==2.4.5
graphviz==0.8.4
idna==2.7
jinja2==2.10
markupsafe==1.0
idna==2.8
jinja2==2.10.1
markupsafe==1.1.1
mondrian==0.8.0
packaging==17.1
pbr==4.2.0
psutil==5.4.6
pyparsing==2.2.0
python-slugify==1.2.5
pytz==2018.5
requests==2.19.1
six==1.11.0
sqlalchemy==1.2.10
stevedore==1.29.0
typing==3.6.4
unidecode==1.0.22
urllib3==1.23
pbr==5.2.0
psutil==5.6.2
pyparsing==2.4.0
python-slugify==1.2.6
pytz==2019.1
requests==2.21.0
six==1.12.0
sqlalchemy==1.3.3
stevedore==1.30.1
unidecode==1.0.23
urllib3==1.24.3
whistle==1.0.1

33
requirements.txt
View File

@ -1,24 +1,23 @@
-e .
appdirs==1.4.3
certifi==2018.4.16
certifi==2019.3.9
chardet==3.0.4
colorama==0.3.9
fs==2.0.27
fs==2.4.5
graphviz==0.8.4
idna==2.7
jinja2==2.10
markupsafe==1.0
idna==2.8
jinja2==2.10.1
markupsafe==1.1.1
mondrian==0.8.0
packaging==17.1
pbr==4.2.0
psutil==5.4.6
pyparsing==2.2.0
python-slugify==1.2.5
pytz==2018.5
requests==2.19.1
six==1.11.0
stevedore==1.29.0
typing==3.6.4
unidecode==1.0.22
urllib3==1.23
packaging==19.0
pbr==5.2.0
psutil==5.6.2
pyparsing==2.4.0
python-slugify==1.2.6
pytz==2019.1
requests==2.21.0
six==1.12.0
stevedore==1.30.1
unidecode==1.0.23
urllib3==1.24.3
whistle==1.0.1

54
setup.py
View File

@ -1,11 +1,12 @@
# Generated by Medikit 0.6.3 on 2018-08-11.
# Generated by Medikit 0.7.1 on 2019-05-16.
# All changes will be overriden.
# Edit Projectfile and run “make update” (or “medikit update”) to regenerate.
from setuptools import setup, find_packages
from codecs import open
from os import path
from setuptools import find_packages, setup
here = path.abspath(path.dirname(__file__))
# Py3 compatibility hacks, borrowed from IPython.
@ -44,14 +45,17 @@ else:
setup(
author='Romain Dorgueil',
author_email='romain@dorgueil.net',
data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [
data_files=[
(
'share/jupyter/nbextensions/bonobo-jupyter',
[
'bonobo/contrib/jupyter/static/extension.js',
'bonobo/contrib/jupyter/static/index.js',
'bonobo/contrib/jupyter/static/index.js.map'
])],
description=(
'Bonobo, a simple, modern and atomic extract-transform-load toolkit for '
'python 3.5+.'),
'bonobo/contrib/jupyter/static/index.js.map',
],
)
],
description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'python 3.5+.'),
license='Apache License, Version 2.0',
name='bonobo',
version=version,
@ -60,21 +64,30 @@ setup(
packages=find_packages(exclude=['ez_setup', 'example', 'test']),
include_package_data=True,
install_requires=[
'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)',
'mondrian (~= 0.7)', 'packaging (~= 17.0)', 'psutil (~= 5.4)',
'python-slugify (~= 1.2.0)', 'requests (~= 2.0)',
'stevedore (~= 1.27)', 'whistle (~= 1.0)'
'fs (~= 2.0)',
'graphviz (>= 0.8, < 0.9)',
'jinja2 (~= 2.9)',
'mondrian (~= 0.7)',
'packaging (~= 19.0)',
'psutil (~= 5.4)',
'python-slugify (~= 1.2.0)',
'requests (~= 2.0)',
'stevedore (~= 1.27)',
'whistle (~= 1.0)',
],
extras_require={
'dev': [
'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)',
'pytest (~= 3.4)', 'pytest-cov (~= 2.5)',
'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)',
'sphinx-sitemap (>= 0.2, < 0.3)'
'cookiecutter (>= 1.5, < 1.6)',
'coverage (~= 4.4)',
'pytest (~= 3.4)',
'pytest-cov (~= 2.5)',
'pytest-timeout (>= 1, < 2)',
'sphinx (~= 1.7)',
'sphinx-sitemap (>= 0.2, < 0.3)',
],
'docker': ['bonobo-docker (~= 0.6.0a1)'],
'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'],
'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)']
'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'],
},
entry_points={
'bonobo.commands': [
@ -84,11 +97,10 @@ setup(
'init = bonobo.commands.init:InitCommand',
'inspect = bonobo.commands.inspect:InspectCommand',
'run = bonobo.commands.run:RunCommand',
'version = bonobo.commands.version:VersionCommand'
'version = bonobo.commands.version:VersionCommand',
],
'console_scripts': ['bonobo = bonobo.commands:entrypoint']
'console_scripts': ['bonobo = bonobo.commands:entrypoint'],
},
url='https://www.bonobo-project.org/',
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.
format(version=version),
download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version),
)