adds yapf for automatic style.
This commit is contained in:
3
.style.yapf
Normal file
3
.style.yapf
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
[style]
|
||||||
|
based_on_style = pep8
|
||||||
|
column_limit = 120
|
||||||
@ -105,10 +105,15 @@ class ComponentExecutionContext(WithStatistics):
|
|||||||
|
|
||||||
def get_stats(self, *args, **kwargs):
|
def get_stats(self, *args, **kwargs):
|
||||||
return (
|
return (
|
||||||
('in', self.stats['in'],),
|
(
|
||||||
('out', self.stats['out'],),
|
'in',
|
||||||
('err', self.stats['err'],),
|
self.stats['in'], ),
|
||||||
)
|
(
|
||||||
|
'out',
|
||||||
|
self.stats['out'], ),
|
||||||
|
(
|
||||||
|
'err',
|
||||||
|
self.stats['err'], ), )
|
||||||
|
|
||||||
def impulse(self):
|
def impulse(self):
|
||||||
self.input.put(None)
|
self.input.put(None)
|
||||||
@ -130,14 +135,13 @@ class ComponentExecutionContext(WithStatistics):
|
|||||||
# timer = Timer()
|
# timer = Timer()
|
||||||
# with timer:
|
# with timer:
|
||||||
|
|
||||||
args = () if row is None else (row,)
|
args = () if row is None else (row, )
|
||||||
if getattr(self.component, '_with_context', False):
|
if getattr(self.component, '_with_context', False):
|
||||||
return self.component(self, *args)
|
return self.component(self, *args)
|
||||||
return self.component(*args)
|
return self.component(*args)
|
||||||
|
|
||||||
def step(self, finalize=False):
|
def step(self, finalize=False):
|
||||||
# Pull data from the first available input channel.
|
# Pull data from the first available input channel.
|
||||||
|
|
||||||
"""Runs a transformation callable with given args/kwargs and flush the result into the right
|
"""Runs a transformation callable with given args/kwargs and flush the result into the right
|
||||||
output channel."""
|
output channel."""
|
||||||
|
|
||||||
|
|||||||
@ -15,15 +15,14 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
|
||||||
|
|
||||||
class AbstractError(NotImplementedError):
|
class AbstractError(NotImplementedError):
|
||||||
"""Abstract error is a convenient error to declare a method as "being left as an exercise for the reader"."""
|
"""Abstract error is a convenient error to declare a method as "being left as an exercise for the reader"."""
|
||||||
|
|
||||||
def __init__(self, method):
|
def __init__(self, method):
|
||||||
super().__init__(
|
super().__init__('Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format(
|
||||||
'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format(
|
class_name=method.__self__.__name__,
|
||||||
class_name=method.__self__.__name__,
|
method_name=method.__name__, ))
|
||||||
method_name=method.__name__,
|
|
||||||
))
|
|
||||||
|
|
||||||
|
|
||||||
class InactiveIOError(IOError):
|
class InactiveIOError(IOError):
|
||||||
@ -42,8 +41,7 @@ class ValidationError(RuntimeError):
|
|||||||
def __init__(self, inst, message):
|
def __init__(self, inst, message):
|
||||||
super(ValidationError, self).__init__('Validation error in {class_name}: {message}'.format(
|
super(ValidationError, self).__init__('Validation error in {class_name}: {message}'.format(
|
||||||
class_name=type(inst).__name__,
|
class_name=type(inst).__name__,
|
||||||
message=message,
|
message=message, ))
|
||||||
))
|
|
||||||
|
|
||||||
|
|
||||||
class ProhibitedOperationError(RuntimeError):
|
class ProhibitedOperationError(RuntimeError):
|
||||||
|
|||||||
@ -84,8 +84,8 @@ class Input(Queue, Readable, Writable):
|
|||||||
self.on_end()
|
self.on_end()
|
||||||
|
|
||||||
if not self.alive:
|
if not self.alive:
|
||||||
raise InactiveReadableError(
|
raise InactiveReadableError('Cannot get() on an inactive {} (runlevel just reached 0).'.format(
|
||||||
'Cannot get() on an inactive {} (runlevel just reached 0).'.format(Readable.__name__))
|
Readable.__name__))
|
||||||
return self.get(block, timeout)
|
return self.get(block, timeout)
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|||||||
@ -23,9 +23,7 @@ class service:
|
|||||||
return item
|
return item
|
||||||
|
|
||||||
def define(self, *args, **kwargs):
|
def define(self, *args, **kwargs):
|
||||||
new_service = type(self)(
|
new_service = type(self)(partial(self.factory, *args, **kwargs))
|
||||||
partial(self.factory, *args, **kwargs)
|
|
||||||
)
|
|
||||||
self.children.add(new_service)
|
self.children.add(new_service)
|
||||||
return new_service
|
return new_service
|
||||||
|
|
||||||
@ -47,13 +45,9 @@ def inject(*iargs, **ikwargs):
|
|||||||
def wrapper(target):
|
def wrapper(target):
|
||||||
@functools.wraps(target)
|
@functools.wraps(target)
|
||||||
def wrapped(*args, **kwargs):
|
def wrapped(*args, **kwargs):
|
||||||
return target(
|
return target(*itertools.chain(map(resolve, iargs), args),
|
||||||
*itertools.chain(map(resolve, iargs), args),
|
**{ ** kwargs, ** {k: resolve(v)
|
||||||
**{
|
for k, v in ikwargs.items()}})
|
||||||
**kwargs,
|
|
||||||
**{k: resolve(v) for k, v in ikwargs.items()}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
return wrapped
|
return wrapped
|
||||||
|
|
||||||
|
|||||||
@ -25,6 +25,4 @@ class WithStatistics(metaclass=ABCMeta):
|
|||||||
raise AbstractError(self.get_stats)
|
raise AbstractError(self.get_stats)
|
||||||
|
|
||||||
def get_stats_as_string(self, *args, **kwargs):
|
def get_stats_as_string(self, *args, **kwargs):
|
||||||
return ' '.join(
|
return ' '.join(('{0}={1}'.format(name, cnt) for name, cnt in self.get_stats(*args, **kwargs) if cnt > 0))
|
||||||
('{0}={1}'.format(name, cnt) for name, cnt in self.get_stats(*args, **kwargs) if cnt > 0)
|
|
||||||
)
|
|
||||||
|
|||||||
@ -12,7 +12,7 @@ class NaiveStrategy(Strategy):
|
|||||||
for i, component in enumerate(context.graph.components):
|
for i, component in enumerate(context.graph.components):
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
args = (input_queues[i].get(block=False),) if i else ()
|
args = (input_queues[i].get(block=False), ) if i else ()
|
||||||
for row in force_iterator(component(*args)):
|
for row in force_iterator(component(*args)):
|
||||||
input_queues[i + 1].put(row)
|
input_queues[i + 1].put(row)
|
||||||
if not i:
|
if not i:
|
||||||
|
|||||||
@ -1,3 +1 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -26,13 +26,14 @@ t = blessings.Terminal()
|
|||||||
@lru_cache(1)
|
@lru_cache(1)
|
||||||
def memory_usage():
|
def memory_usage():
|
||||||
process = psutil.Process(os.getpid())
|
process = psutil.Process(os.getpid())
|
||||||
return process.get_memory_info()[0] / float(2 ** 20)
|
return process.get_memory_info()[0] / float(2**20)
|
||||||
|
|
||||||
|
|
||||||
# @lru_cache(64)
|
# @lru_cache(64)
|
||||||
# def execution_time(harness):
|
# def execution_time(harness):
|
||||||
# return datetime.datetime.now() - harness._started_at
|
# return datetime.datetime.now() - harness._started_at
|
||||||
|
|
||||||
|
|
||||||
class ConsoleOutputPlugin:
|
class ConsoleOutputPlugin:
|
||||||
"""
|
"""
|
||||||
Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor
|
Outputs status information to the connected stdout. Can be a TTY, with or without support for colors/cursor
|
||||||
@ -50,10 +51,9 @@ class ConsoleOutputPlugin:
|
|||||||
def _write(self, context, rewind):
|
def _write(self, context, rewind):
|
||||||
profile, debug = False, False
|
profile, debug = False, False
|
||||||
if profile:
|
if profile:
|
||||||
append = (
|
append = (('Memory', '{0:.2f} Mb'.format(memory_usage())),
|
||||||
('Memory', '{0:.2f} Mb'.format(memory_usage())),
|
# ('Total time', '{0} s'.format(execution_time(harness))),
|
||||||
# ('Total time', '{0} s'.format(execution_time(harness))),
|
)
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
append = ()
|
append = ()
|
||||||
self.write(context, prefix=self.prefix, append=append, debug=debug, profile=profile, rewind=rewind)
|
self.write(context, prefix=self.prefix, append=append, debug=debug, profile=profile, rewind=rewind)
|
||||||
@ -76,24 +76,24 @@ class ConsoleOutputPlugin:
|
|||||||
for i, component in enumerate(context):
|
for i, component in enumerate(context):
|
||||||
if component.running:
|
if component.running:
|
||||||
_line = ''.join((
|
_line = ''.join((
|
||||||
t.black('({})'.format(i+1)),
|
t.black('({})'.format(i + 1)),
|
||||||
' ',
|
' ',
|
||||||
t.bold(t.white('+')),
|
t.bold(t.white('+')),
|
||||||
' ',
|
' ',
|
||||||
component.name,
|
component.name,
|
||||||
' ',
|
' ',
|
||||||
component.get_stats_as_string(debug=debug, profile=profile),
|
component.get_stats_as_string(
|
||||||
' ',
|
debug=debug, profile=profile),
|
||||||
))
|
' ', ))
|
||||||
else:
|
else:
|
||||||
_line = t.black(''.join((
|
_line = t.black(''.join((
|
||||||
'({})'.format(i+1),
|
'({})'.format(i + 1),
|
||||||
' - ',
|
' - ',
|
||||||
component.name,
|
component.name,
|
||||||
' ',
|
' ',
|
||||||
component.get_stats_as_string(debug=debug, profile=profile),
|
component.get_stats_as_string(
|
||||||
' ',
|
debug=debug, profile=profile),
|
||||||
)))
|
' ', )))
|
||||||
print(prefix + _line + t.clear_eol)
|
print(prefix + _line + t.clear_eol)
|
||||||
|
|
||||||
if append:
|
if append:
|
||||||
|
|||||||
@ -17,7 +17,9 @@ from bonobo import service
|
|||||||
@service
|
@service
|
||||||
def client(username, password):
|
def client(username, password):
|
||||||
client = couchdb.Server()
|
client = couchdb.Server()
|
||||||
client.resource.credentials = (username, password,)
|
client.resource.credentials = (
|
||||||
|
username,
|
||||||
|
password, )
|
||||||
return client
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -3,12 +3,7 @@ from .plugin import JupyterOutputPlugin
|
|||||||
|
|
||||||
|
|
||||||
def _jupyter_nbextension_paths():
|
def _jupyter_nbextension_paths():
|
||||||
return [{
|
return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}]
|
||||||
'section': 'notebook',
|
|
||||||
'src': 'static',
|
|
||||||
'dest': 'bonobo-jupyter',
|
|
||||||
'require': 'bonobo-jupyter/extension'
|
|
||||||
}]
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
@ -4,7 +4,9 @@ import requests # todo: make this a service so we can substitute it ?
|
|||||||
|
|
||||||
|
|
||||||
def extract_ods(url, dataset, rows=100, **kwargs):
|
def extract_ods(url, dataset, rows=100, **kwargs):
|
||||||
params = (('dataset', dataset), ('rows', rows),) + tuple(sorted(kwargs.items()))
|
params = (
|
||||||
|
('dataset', dataset),
|
||||||
|
('rows', rows), ) + tuple(sorted(kwargs.items()))
|
||||||
base_url = url + '?' + urlencode(params)
|
base_url = url + '?' + urlencode(params)
|
||||||
|
|
||||||
def _extract_ods():
|
def _extract_ods():
|
||||||
@ -18,10 +20,7 @@ def extract_ods(url, dataset, rows=100, **kwargs):
|
|||||||
break
|
break
|
||||||
|
|
||||||
for row in records:
|
for row in records:
|
||||||
yield {
|
yield { ** row.get('fields', {}), 'geometry': row.get('geometry', {})}
|
||||||
**row.get('fields', {}),
|
|
||||||
'geometry': row.get('geometry', {})
|
|
||||||
}
|
|
||||||
|
|
||||||
start += rows
|
start += rows
|
||||||
|
|
||||||
|
|||||||
@ -12,4 +12,4 @@ BEGIN = Token('Begin')
|
|||||||
END = Token('End')
|
END = Token('End')
|
||||||
NEW = Token('New')
|
NEW = Token('New')
|
||||||
RUNNING = Token('Running')
|
RUNNING = Token('Running')
|
||||||
TERMINATED = Token('Terminated')
|
TERMINATED = Token('Terminated')
|
||||||
|
|||||||
Reference in New Issue
Block a user