paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adds yapf for automatic style.

Browse Source
This commit is contained in:
Romain Dorgueil
2016-12-24 10:57:47 +01:00
parent 14d6d2c0dd
commit b1880b42f9
13 changed files with 48 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.style.yapf Normal file
View File

@ -0,0 +1,3 @@
[style]
based_on_style = pep8
column_limit = 120

16
bonobo/core/contexts.py
View File

@ -105,10 +105,15 @@ class ComponentExecutionContext(WithStatistics):
def get_stats(self, *args, **kwargs):
return (
('in', self.stats['in'],),
('out', self.stats['out'],),
('err', self.stats['err'],),
)
(
'in',
self.stats['in'], ),
(
'out',
self.stats['out'], ),
(
'err',
self.stats['err'], ), )
def impulse(self):
self.input.put(None)
@ -130,14 +135,13 @@ class ComponentExecutionContext(WithStatistics):
# timer = Timer()
# with timer:
args = () if row is None else (row,)
args = () if row is None else (row, )
if getattr(self.component, '_with_context', False):
return self.component(self, *args)
return self.component(*args)
def step(self, finalize=False):
# Pull data from the first available input channel.
"""Runs a transformation callable with given args/kwargs and flush the result into the right
output channel."""

10
bonobo/core/errors.py
View File

@ -15,15 +15,14 @@
# limitations under the License.
#
class AbstractError(NotImplementedError):
"""Abstract error is a convenient error to declare a method as "being left as an exercise for the reader"."""
def __init__(self, method):
super().__init__(
'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format(
super().__init__('Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format(
class_name=method.__self__.__name__,
method_name=method.__name__,
))
method_name=method.__name__, ))
class InactiveIOError(IOError):
@ -42,8 +41,7 @@ class ValidationError(RuntimeError):
def __init__(self, inst, message):
super(ValidationError, self).__init__('Validation error in {class_name}: {message}'.format(
class_name=type(inst).__name__,
message=message,
))
message=message, ))
class ProhibitedOperationError(RuntimeError):

4
bonobo/core/inputs.py
View File

@ -84,8 +84,8 @@ class Input(Queue, Readable, Writable):
self.on_end()
if not self.alive:
raise InactiveReadableError(
'Cannot get() on an inactive {} (runlevel just reached 0).'.format(Readable.__name__))
raise InactiveReadableError('Cannot get() on an inactive {} (runlevel just reached 0).'.format(
Readable.__name__))
return self.get(block, timeout)
return data

14
bonobo/core/services.py
View File

@ -23,9 +23,7 @@ class service:
return item
def define(self, *args, **kwargs):
new_service = type(self)(
partial(self.factory, *args, **kwargs)
)
new_service = type(self)(partial(self.factory, *args, **kwargs))
self.children.add(new_service)
return new_service
@ -47,13 +45,9 @@ def inject(*iargs, **ikwargs):
def wrapper(target):
@functools.wraps(target)
def wrapped(*args, **kwargs):
return target(
*itertools.chain(map(resolve, iargs), args),
**{
**kwargs,
**{k: resolve(v) for k, v in ikwargs.items()}
}
)
return target(*itertools.chain(map(resolve, iargs), args),
**{ ** kwargs, ** {k: resolve(v)
for k, v in ikwargs.items()}})
return wrapped

4
bonobo/core/stats.py
View File

@ -25,6 +25,4 @@ class WithStatistics(metaclass=ABCMeta):
raise AbstractError(self.get_stats)
def get_stats_as_string(self, *args, **kwargs):
return ' '.join(
('{0}={1}'.format(name, cnt) for name, cnt in self.get_stats(*args, **kwargs) if cnt > 0)
)
return ' '.join(('{0}={1}'.format(name, cnt) for name, cnt in self.get_stats(*args, **kwargs) if cnt > 0))

2
bonobo/core/strategies/naive.py
View File

@ -12,7 +12,7 @@ class NaiveStrategy(Strategy):
for i, component in enumerate(context.graph.components):
while True:
try:
args = (input_queues[i].get(block=False),) if i else ()
args = (input_queues[i].get(block=False), ) if i else ()
for row in force_iterator(component(*args)):
input_queues[i + 1].put(row)
if not i:

2
bonobo/core/strategies/util.py
View File

@ -1,3 +1 @@

22
bonobo/ext/console/plugin.py
View File

@ -26,13 +26,14 @@ t = blessings.Terminal()
@lru_cache(1)
def memory_usage():
process = psutil.Process(os.getpid())
return process.get_memory_info()[0] / float(2 ** 20)
return process.get_memory_info()[0] / float(2**20)
# @lru_cache(64)
# def execution_time(harness):
# return datetime.datetime.now() - harness._started_at
class ConsoleOutputPlugin:
"""
Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor
@ -50,8 +51,7 @@ class ConsoleOutputPlugin:
def _write(self, context, rewind):
profile, debug = False, False
if profile:
append = (
('Memory', '{0:.2f} Mb'.format(memory_usage())),
append = (('Memory', '{0:.2f} Mb'.format(memory_usage())),
# ('Total time', '{0} s'.format(execution_time(harness))),
)
else:
@ -76,24 +76,24 @@ class ConsoleOutputPlugin:
for i, component in enumerate(context):
if component.running:
_line = ''.join((
t.black('({})'.format(i+1)),
t.black('({})'.format(i + 1)),
' ',
t.bold(t.white('+')),
' ',
component.name,
' ',
component.get_stats_as_string(debug=debug, profile=profile),
' ',
))
component.get_stats_as_string(
debug=debug, profile=profile),
' ', ))
else:
_line = t.black(''.join((
'({})'.format(i+1),
'({})'.format(i + 1),
' - ',
component.name,
' ',
component.get_stats_as_string(debug=debug, profile=profile),
' ',
)))
component.get_stats_as_string(
debug=debug, profile=profile),
' ', )))
print(prefix + _line + t.clear_eol)
if append:

4
bonobo/ext/couchdb_.py
View File

@ -17,7 +17,9 @@ from bonobo import service
@service
def client(username, password):
client = couchdb.Server()
client.resource.credentials = (username, password,)
client.resource.credentials = (
username,
password, )
return client

7
bonobo/ext/jupyter/__init__.py
View File

@ -3,12 +3,7 @@ from .plugin import JupyterOutputPlugin
def _jupyter_nbextension_paths():
return [{
'section': 'notebook',
'src': 'static',
'dest': 'bonobo-jupyter',
'require': 'bonobo-jupyter/extension'
}]
return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}]
__all__ = [

9
bonobo/ext/ods.py
View File

@ -4,7 +4,9 @@ import requests # todo: make this a service so we can substitute it ?
def extract_ods(url, dataset, rows=100, **kwargs):
params = (('dataset', dataset), ('rows', rows),) + tuple(sorted(kwargs.items()))
params = (
('dataset', dataset),
('rows', rows), ) + tuple(sorted(kwargs.items()))
base_url = url + '?' + urlencode(params)
def _extract_ods():
@ -18,10 +20,7 @@ def extract_ods(url, dataset, rows=100, **kwargs):
break
for row in records:
yield {
**row.get('fields', {}),
'geometry': row.get('geometry', {})
}
yield { ** row.get('fields', {}), 'geometry': row.get('geometry', {})}
start += rows