adds yapf for automatic style.

This commit is contained in:
Romain Dorgueil
2016-12-24 10:57:47 +01:00
parent 14d6d2c0dd
commit b1880b42f9
13 changed files with 48 additions and 57 deletions

3
.style.yapf Normal file
View File

@ -0,0 +1,3 @@
[style]
based_on_style = pep8
column_limit = 120

View File

@ -105,10 +105,15 @@ class ComponentExecutionContext(WithStatistics):
def get_stats(self, *args, **kwargs): def get_stats(self, *args, **kwargs):
return ( return (
('in', self.stats['in'],), (
('out', self.stats['out'],), 'in',
('err', self.stats['err'],), self.stats['in'], ),
) (
'out',
self.stats['out'], ),
(
'err',
self.stats['err'], ), )
def impulse(self): def impulse(self):
self.input.put(None) self.input.put(None)
@ -130,14 +135,13 @@ class ComponentExecutionContext(WithStatistics):
# timer = Timer() # timer = Timer()
# with timer: # with timer:
args = () if row is None else (row,) args = () if row is None else (row, )
if getattr(self.component, '_with_context', False): if getattr(self.component, '_with_context', False):
return self.component(self, *args) return self.component(self, *args)
return self.component(*args) return self.component(*args)
def step(self, finalize=False): def step(self, finalize=False):
# Pull data from the first available input channel. # Pull data from the first available input channel.
"""Runs a transformation callable with given args/kwargs and flush the result into the right """Runs a transformation callable with given args/kwargs and flush the result into the right
output channel.""" output channel."""

View File

@ -15,15 +15,14 @@
# limitations under the License. # limitations under the License.
# #
class AbstractError(NotImplementedError): class AbstractError(NotImplementedError):
"""Abstract error is a convenient error to declare a method as "being left as an exercise for the reader".""" """Abstract error is a convenient error to declare a method as "being left as an exercise for the reader"."""
def __init__(self, method): def __init__(self, method):
super().__init__( super().__init__('Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format(
'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format( class_name=method.__self__.__name__,
class_name=method.__self__.__name__, method_name=method.__name__, ))
method_name=method.__name__,
))
class InactiveIOError(IOError): class InactiveIOError(IOError):
@ -42,8 +41,7 @@ class ValidationError(RuntimeError):
def __init__(self, inst, message): def __init__(self, inst, message):
super(ValidationError, self).__init__('Validation error in {class_name}: {message}'.format( super(ValidationError, self).__init__('Validation error in {class_name}: {message}'.format(
class_name=type(inst).__name__, class_name=type(inst).__name__,
message=message, message=message, ))
))
class ProhibitedOperationError(RuntimeError): class ProhibitedOperationError(RuntimeError):

View File

@ -84,8 +84,8 @@ class Input(Queue, Readable, Writable):
self.on_end() self.on_end()
if not self.alive: if not self.alive:
raise InactiveReadableError( raise InactiveReadableError('Cannot get() on an inactive {} (runlevel just reached 0).'.format(
'Cannot get() on an inactive {} (runlevel just reached 0).'.format(Readable.__name__)) Readable.__name__))
return self.get(block, timeout) return self.get(block, timeout)
return data return data

View File

@ -23,9 +23,7 @@ class service:
return item return item
def define(self, *args, **kwargs): def define(self, *args, **kwargs):
new_service = type(self)( new_service = type(self)(partial(self.factory, *args, **kwargs))
partial(self.factory, *args, **kwargs)
)
self.children.add(new_service) self.children.add(new_service)
return new_service return new_service
@ -47,13 +45,9 @@ def inject(*iargs, **ikwargs):
def wrapper(target): def wrapper(target):
@functools.wraps(target) @functools.wraps(target)
def wrapped(*args, **kwargs): def wrapped(*args, **kwargs):
return target( return target(*itertools.chain(map(resolve, iargs), args),
*itertools.chain(map(resolve, iargs), args), **{ ** kwargs, ** {k: resolve(v)
**{ for k, v in ikwargs.items()}})
**kwargs,
**{k: resolve(v) for k, v in ikwargs.items()}
}
)
return wrapped return wrapped

View File

@ -25,6 +25,4 @@ class WithStatistics(metaclass=ABCMeta):
raise AbstractError(self.get_stats) raise AbstractError(self.get_stats)
def get_stats_as_string(self, *args, **kwargs): def get_stats_as_string(self, *args, **kwargs):
return ' '.join( return ' '.join(('{0}={1}'.format(name, cnt) for name, cnt in self.get_stats(*args, **kwargs) if cnt > 0))
('{0}={1}'.format(name, cnt) for name, cnt in self.get_stats(*args, **kwargs) if cnt > 0)
)

View File

@ -12,7 +12,7 @@ class NaiveStrategy(Strategy):
for i, component in enumerate(context.graph.components): for i, component in enumerate(context.graph.components):
while True: while True:
try: try:
args = (input_queues[i].get(block=False),) if i else () args = (input_queues[i].get(block=False), ) if i else ()
for row in force_iterator(component(*args)): for row in force_iterator(component(*args)):
input_queues[i + 1].put(row) input_queues[i + 1].put(row)
if not i: if not i:

View File

@ -1,3 +1 @@

View File

@ -26,13 +26,14 @@ t = blessings.Terminal()
@lru_cache(1) @lru_cache(1)
def memory_usage(): def memory_usage():
process = psutil.Process(os.getpid()) process = psutil.Process(os.getpid())
return process.get_memory_info()[0] / float(2 ** 20) return process.get_memory_info()[0] / float(2**20)
# @lru_cache(64) # @lru_cache(64)
# def execution_time(harness): # def execution_time(harness):
# return datetime.datetime.now() - harness._started_at # return datetime.datetime.now() - harness._started_at
class ConsoleOutputPlugin: class ConsoleOutputPlugin:
""" """
Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor
@ -50,10 +51,9 @@ class ConsoleOutputPlugin:
def _write(self, context, rewind): def _write(self, context, rewind):
profile, debug = False, False profile, debug = False, False
if profile: if profile:
append = ( append = (('Memory', '{0:.2f} Mb'.format(memory_usage())),
('Memory', '{0:.2f} Mb'.format(memory_usage())), # ('Total time', '{0} s'.format(execution_time(harness))),
# ('Total time', '{0} s'.format(execution_time(harness))), )
)
else: else:
append = () append = ()
self.write(context, prefix=self.prefix, append=append, debug=debug, profile=profile, rewind=rewind) self.write(context, prefix=self.prefix, append=append, debug=debug, profile=profile, rewind=rewind)
@ -76,24 +76,24 @@ class ConsoleOutputPlugin:
for i, component in enumerate(context): for i, component in enumerate(context):
if component.running: if component.running:
_line = ''.join(( _line = ''.join((
t.black('({})'.format(i+1)), t.black('({})'.format(i + 1)),
' ', ' ',
t.bold(t.white('+')), t.bold(t.white('+')),
' ', ' ',
component.name, component.name,
' ', ' ',
component.get_stats_as_string(debug=debug, profile=profile), component.get_stats_as_string(
' ', debug=debug, profile=profile),
)) ' ', ))
else: else:
_line = t.black(''.join(( _line = t.black(''.join((
'({})'.format(i+1), '({})'.format(i + 1),
' - ', ' - ',
component.name, component.name,
' ', ' ',
component.get_stats_as_string(debug=debug, profile=profile), component.get_stats_as_string(
' ', debug=debug, profile=profile),
))) ' ', )))
print(prefix + _line + t.clear_eol) print(prefix + _line + t.clear_eol)
if append: if append:

View File

@ -17,7 +17,9 @@ from bonobo import service
@service @service
def client(username, password): def client(username, password):
client = couchdb.Server() client = couchdb.Server()
client.resource.credentials = (username, password,) client.resource.credentials = (
username,
password, )
return client return client

View File

@ -3,12 +3,7 @@ from .plugin import JupyterOutputPlugin
def _jupyter_nbextension_paths(): def _jupyter_nbextension_paths():
return [{ return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}]
'section': 'notebook',
'src': 'static',
'dest': 'bonobo-jupyter',
'require': 'bonobo-jupyter/extension'
}]
__all__ = [ __all__ = [

View File

@ -4,7 +4,9 @@ import requests # todo: make this a service so we can substitute it ?
def extract_ods(url, dataset, rows=100, **kwargs): def extract_ods(url, dataset, rows=100, **kwargs):
params = (('dataset', dataset), ('rows', rows),) + tuple(sorted(kwargs.items())) params = (
('dataset', dataset),
('rows', rows), ) + tuple(sorted(kwargs.items()))
base_url = url + '?' + urlencode(params) base_url = url + '?' + urlencode(params)
def _extract_ods(): def _extract_ods():
@ -18,10 +20,7 @@ def extract_ods(url, dataset, rows=100, **kwargs):
break break
for row in records: for row in records:
yield { yield { ** row.get('fields', {}), 'geometry': row.get('geometry', {})}
**row.get('fields', {}),
'geometry': row.get('geometry', {})
}
start += rows start += rows

View File

@ -12,4 +12,4 @@ BEGIN = Token('Begin')
END = Token('End') END = Token('End')
NEW = Token('New') NEW = Token('New')
RUNNING = Token('Running') RUNNING = Token('Running')
TERMINATED = Token('Terminated') TERMINATED = Token('Terminated')