From b035bdea325905078657bf7c498bfdffdb3a43c1 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 12 Feb 2017 08:10:22 +0100 Subject: [PATCH] formating, better consistency in readers, ability to read files from http (fast and dirty). --- bonobo/__init__.py | 6 +-- bonobo/commands/__init__.py | 5 +- bonobo/commands/run.py | 7 +-- bonobo/config.py | 5 +- bonobo/context/execution.py | 5 +- bonobo/core/bags.py | 5 +- bonobo/core/strategies/executor.py | 2 + bonobo/examples/datasets/coffeeshops.py | 17 ++++--- bonobo/examples/datasets/fablabs.py | 9 ++-- bonobo/examples/datasets/passwd.txt | 30 ++++++++++++ bonobo/examples/files/json.py | 8 ++++ bonobo/examples/files/text.py | 4 +- bonobo/examples/types/bags.py | 6 +-- bonobo/examples/types/dicts.py | 6 +-- bonobo/examples/types/strings.py | 6 +-- bonobo/ext/console/__init__.py | 4 +- bonobo/ext/console/plugin.py | 22 ++------- bonobo/ext/jupyter/__init__.py | 4 +- bonobo/ext/opendatasoft.py | 62 ++++++++++++++++--------- bonobo/io/csv.py | 5 +- bonobo/io/file.py | 12 ++++- bonobo/io/json.py | 8 +++- bonobo/util/__init__.py | 5 +- bonobo/util/compat.py | 19 ++++++++ bonobo/util/objects.py | 2 +- docs/conf.py | 9 ++-- examples | 1 - setup.py | 36 +++++++------- tests/core/test_bags.py | 26 ++--------- tests/core/test_statistics.py | 5 +- tests/ext/test_ods.py | 11 +++-- tests/test_basicusage.py | 4 +- tests/test_commands.py | 5 +- 33 files changed, 203 insertions(+), 158 deletions(-) create mode 100644 bonobo/examples/datasets/passwd.txt create mode 100644 bonobo/examples/files/json.py delete mode 120000 examples diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 62df11d..f439f18 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -4,7 +4,6 @@ # transformations using a simple directed graph of python callables. # # Licensed under Apache License 2.0, read the LICENSE file in the root of the source tree. - """Bonobo data-processing toolkit main module.""" import sys @@ -62,8 +61,9 @@ def create_strategy(name=None): try: factory = STRATEGIES[name] except KeyError as exc: - raise RuntimeError('Invalid strategy {}. Available choices: {}.'.format(repr(name), ', '.join( - sorted(STRATEGIES.keys())))) from exc + raise RuntimeError( + 'Invalid strategy {}. Available choices: {}.'.format(repr(name), ', '.join(sorted(STRATEGIES.keys()))) + ) from exc return factory() diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index 6525ab6..37d55b1 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -11,6 +11,7 @@ def entrypoint(args=None): subparsers.required = True commands = {} + def register_extension(ext, commands=commands): try: parser = subparsers.add_parser(ext.name) @@ -18,7 +19,9 @@ def entrypoint(args=None): except Exception: logging.exception('Error while loading command {}.'.format(ext.name)) - mgr = ExtensionManager(namespace='bonobo.commands', ) + mgr = ExtensionManager( + namespace='bonobo.commands', + ) mgr.map(register_extension) args = parser.parse_args(args).__dict__ diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 93f81d9..018041a 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -24,9 +24,10 @@ def execute(file, quiet=False): graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph)) - assert len(graphs) == 1, ('Having zero or more than one graph definition in one file is unsupported for now, ' - 'but it is something that will be implemented in the future.\n\nExpected: 1, got: {}.').format( - len(graphs)) + assert len(graphs) == 1, ( + 'Having zero or more than one graph definition in one file is unsupported for now, ' + 'but it is something that will be implemented in the future.\n\nExpected: 1, got: {}.' + ).format(len(graphs)) name, graph = list(graphs.items())[0] diff --git a/bonobo/config.py b/bonobo/config.py index 1a0f3ca..4456e0d 100644 --- a/bonobo/config.py +++ b/bonobo/config.py @@ -3,6 +3,7 @@ __all__ = [ 'Option', ] + class Option: def __init__(self, type=None, *, required=False, default=None): self.name = None @@ -11,7 +12,9 @@ class Option: self.default = default def __get__(self, inst, typ): - return inst.__options_values__.get(self.name, self.default) + if not self.name in inst.__options_values__: + inst.__options_values__[self.name] = self.default() if callable(self.default) else self.default + return inst.__options_values__[self.name] def __set__(self, inst, value): inst.__options_values__[self.name] = self.type(value) if self.type else value diff --git a/bonobo/context/execution.py b/bonobo/context/execution.py index 5e94922..2f23318 100644 --- a/bonobo/context/execution.py +++ b/bonobo/context/execution.py @@ -101,8 +101,8 @@ class LoopingExecutionContext(Wrapper): self._started, self._stopped, self._context, self._stack = False, False, None, [] def start(self): - assert self.state == (False, False), ('{}.start() can only be called on a new node.' - ).format(type(self).__name__) + assert self.state == (False, + False), ('{}.start() can only be called on a new node.').format(type(self).__name__) assert self._context is None self._started = True @@ -175,6 +175,7 @@ class PluginExecutionContext(LoopingExecutionContext): LoopingExecutionContext.__init__(self, wrapped, parent) def shutdown(self): + self.wrapped.finalize(self) self.alive = False def step(self): diff --git a/bonobo/core/bags.py b/bonobo/core/bags.py index 9ef8bb8..ca40f61 100644 --- a/bonobo/core/bags.py +++ b/bonobo/core/bags.py @@ -21,10 +21,7 @@ class Bag: def args(self): if self._parent is None: return self._args - return ( - *self._parent.args, - *self._args, - ) + return (*self._parent.args, *self._args, ) @property def kwargs(self): diff --git a/bonobo/core/strategies/executor.py b/bonobo/core/strategies/executor.py index eb45ab1..6be536b 100644 --- a/bonobo/core/strategies/executor.py +++ b/bonobo/core/strategies/executor.py @@ -39,9 +39,11 @@ class ExecutorStrategy(Strategy): futures.append(executor.submit(_runner)) for node_context in context.nodes: + def _runner(node_context=node_context): node_context.start() node_context.loop() + futures.append(executor.submit(_runner)) while context.alive: diff --git a/bonobo/examples/datasets/coffeeshops.py b/bonobo/examples/datasets/coffeeshops.py index 0ce910f..7e801d9 100644 --- a/bonobo/examples/datasets/coffeeshops.py +++ b/bonobo/examples/datasets/coffeeshops.py @@ -1,17 +1,16 @@ from os.path import dirname, realpath, join -from bonobo import console_run -from bonobo.ext.opendatasoft import from_opendatasoft_api -from bonobo.io.file import FileWriter +import bonobo +from bonobo.ext.opendatasoft import OpenDataSoftAPI OUTPUT_FILENAME = realpath(join(dirname(__file__), 'coffeeshops.txt')) -console_run( - from_opendatasoft_api( - 'liste-des-cafes-a-un-euro', netloc='opendata.paris.fr' - ), +graph = bonobo.Graph( + OpenDataSoftAPI(dataset='liste-des-cafes-a-un-euro', netloc='opendata.paris.fr'), lambda row: '{nom_du_cafe}, {adresse}, {arrondissement} Paris, France'.format(**row), - FileWriter(OUTPUT_FILENAME), + bonobo.FileWriter(path=OUTPUT_FILENAME), ) -print('Import done, read {} for results.'.format(OUTPUT_FILENAME)) +if __name__ == '__main__': + bonobo.run(graph) + print('Import done, read {} for results.'.format(OUTPUT_FILENAME)) diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index b04e378..a228f53 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -4,7 +4,7 @@ import os from blessings import Terminal from bonobo import Tee, JsonWriter, Graph, get_examples_path -from bonobo.ext.opendatasoft import from_opendatasoft_api +from bonobo.ext.opendatasoft import OpenDataSoftAPI try: import pycountry @@ -44,8 +44,7 @@ def display(row): address = list( filter( None, ( - ' '.join(filter(None, (row.get('postal_code', None), row.get('city', None)))), - row.get('county', None), + ' '.join(filter(None, (row.get('postal_code', None), row.get('city', None)))), row.get('county', None), row.get('country'), ) ) @@ -58,9 +57,7 @@ def display(row): graph = Graph( - from_opendatasoft_api( - API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris' - ), + OpenDataSoftAPI(dataset=API_DATASET, netloc=API_NETLOC, timezone='Europe/Paris'), normalize, filter_france, Tee(display), diff --git a/bonobo/examples/datasets/passwd.txt b/bonobo/examples/datasets/passwd.txt new file mode 100644 index 0000000..f4695c4 --- /dev/null +++ b/bonobo/examples/datasets/passwd.txt @@ -0,0 +1,30 @@ +root:x:0:0:root:/root:/bin/bash +daemon:x:105:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +sys:x:3:3:sys:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +games:x:5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin +mail:x:0:8:mail:/var/mail:/usr/sbin/nologin +news:x:9:9:news:/var/spool/news:/usr/sbin/nologin +uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin +proxy:x:13:13:proxy:/bin:/usr/sbin/nologin +www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin +backup:x:33:34:backup:/var/backups:/usr/sbin/nologin +list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin +irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin +gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin +systemd-timesync:x:33:103:systemd Time Synchronization,,,:/run/systemd:/bin/false +systemd-network:x:101:104:systemd Network Management,,,:/run/systemd/netif:/bin/false +systemd-resolve:x:102:105:systemd Resolver,,,:/run/systemd/resolve:/bin/false +systemd-bus-proxy:x:103:106:systemd Bus Proxy,,,:/run/systemd:/bin/false +sshd:x:104:65534::/var/run/sshd:/usr/sbin/nologin +ntp:x:105:110::/home/ntp:/bin/false +postfix:x:105:112::/var/spool/postfix:/bin/false +messagebus:x:107:114::/var/run/dbus:/bin/false +debian-security-support:x:108:115:Debian security support check,,,:/var/lib/debian-security-support:/bin/false +snmp:x:109:116::/var/lib/snmp:/usr/sbin/nologin +postgres:x:105:117:PostgreSQL administrator,,,:/var/lib/postgresql:/bin/bash +redis:x:111:118::/var/lib/redis:/bin/false \ No newline at end of file diff --git a/bonobo/examples/files/json.py b/bonobo/examples/files/json.py new file mode 100644 index 0000000..6f9f418 --- /dev/null +++ b/bonobo/examples/files/json.py @@ -0,0 +1,8 @@ +import bonobo as bb + +url = 'https://data.toulouse-metropole.fr/explore/dataset/theatres-et-salles-de-spectacles/download?format=json&timezone=Europe/Berlin&use_labels_for_header=true' + +graph = bb.Graph(bb.JsonReader(path=url), print) + +if __name__ == '__main__': + bb.run(graph) diff --git a/bonobo/examples/files/text.py b/bonobo/examples/files/text.py index a56e7e7..62e5aba 100644 --- a/bonobo/examples/files/text.py +++ b/bonobo/examples/files/text.py @@ -1,4 +1,4 @@ -from bonobo import FileReader, Graph +from bonobo import FileReader, Graph, get_examples_path def skip_comments(line): @@ -7,7 +7,7 @@ def skip_comments(line): graph = Graph( - FileReader(path='/etc/passwd'), + FileReader(path=get_examples_path('datasets/passwd.txt')), skip_comments, lambda s: s.split(':'), lambda l: l[0], diff --git a/bonobo/examples/types/bags.py b/bonobo/examples/types/bags.py index e0609bf..2bfe5de 100644 --- a/bonobo/examples/types/bags.py +++ b/bonobo/examples/types/bags.py @@ -13,7 +13,6 @@ Example on how to use :class:`bonobo.Bag` instances to pass flexible args/kwargs """ - from random import randint from bonobo import Bag, Graph @@ -26,10 +25,7 @@ def extract(): def transform(topic: str): - return Bag.inherit( - title=topic.title(), - rand=randint(10, 99) - ) + return Bag.inherit(title=topic.title(), rand=randint(10, 99)) def load(topic: str, title: str, rand: int): diff --git a/bonobo/examples/types/dicts.py b/bonobo/examples/types/dicts.py index 0e45630..fde4b08 100644 --- a/bonobo/examples/types/dicts.py +++ b/bonobo/examples/types/dicts.py @@ -35,11 +35,7 @@ def load(row: dict): print(row) -graph = Graph( - extract, - transform, - load -) +graph = Graph(extract, transform, load) if __name__ == '__main__': from bonobo import run diff --git a/bonobo/examples/types/strings.py b/bonobo/examples/types/strings.py index 75cfed7..1903151 100644 --- a/bonobo/examples/types/strings.py +++ b/bonobo/examples/types/strings.py @@ -31,11 +31,7 @@ def load(s: str): print(s) -graph = Graph( - extract, - transform, - load -) +graph = Graph(extract, transform, load) if __name__ == '__main__': from bonobo import run diff --git a/bonobo/ext/console/__init__.py b/bonobo/ext/console/__init__.py index 2fffb8f..6770e35 100644 --- a/bonobo/ext/console/__init__.py +++ b/bonobo/ext/console/__init__.py @@ -1,3 +1,5 @@ from .plugin import ConsoleOutputPlugin -__all__ = ['ConsoleOutputPlugin', ] +__all__ = [ + 'ConsoleOutputPlugin', +] diff --git a/bonobo/ext/console/plugin.py b/bonobo/ext/console/plugin.py index c6ee3fc..94bda40 100644 --- a/bonobo/ext/console/plugin.py +++ b/bonobo/ext/console/plugin.py @@ -80,30 +80,16 @@ class ConsoleOutputPlugin(Plugin): if component.alive: _line = ''.join( ( - t.black('({})'.format(i + 1)), - ' ', - t.bold(t.white('+')), - ' ', - component.name, - ' ', - component.get_statistics_as_string( - debug=debug, profile=profile - ), - ' ', + t.black('({})'.format(i + 1)), ' ', t.bold(t.white('+')), ' ', component.name, ' ', + component.get_statistics_as_string(debug=debug, profile=profile), ' ', ) ) else: _line = t.black( ''.join( ( - '({})'.format(i + 1), - ' - ', - component.name, - ' ', - component.get_statistics_as_string( - debug=debug, profile=profile - ), - ' ', + '({})'.format(i + 1), ' - ', component.name, ' ', + component.get_statistics_as_string(debug=debug, profile=profile), ' ', ) ) ) diff --git a/bonobo/ext/jupyter/__init__.py b/bonobo/ext/jupyter/__init__.py index 4ad081d..3ae8068 100644 --- a/bonobo/ext/jupyter/__init__.py +++ b/bonobo/ext/jupyter/__init__.py @@ -5,4 +5,6 @@ def _jupyter_nbextension_paths(): return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}] -__all__ = ['JupyterOutputPlugin', ] +__all__ = [ + 'JupyterOutputPlugin', +] diff --git a/bonobo/ext/opendatasoft.py b/bonobo/ext/opendatasoft.py index 376b5d7..71b3600 100644 --- a/bonobo/ext/opendatasoft.py +++ b/bonobo/ext/opendatasoft.py @@ -2,28 +2,39 @@ from urllib.parse import urlencode import requests # todo: make this a service so we can substitute it ? +from bonobo.config import Configurable, Option +from bonobo.context import ContextProcessor, contextual +from bonobo.util.compat import deprecated +from bonobo.util.objects import ValueHolder -def from_opendatasoft_api( - dataset=None, - endpoint='{scheme}://{netloc}{path}', - scheme='https', - netloc='data.opendatasoft.com', - path='/api/records/1.0/search/', - rows=100, - **kwargs -): - path = path if path.startswith('/') else '/' + path - params = ( - ('dataset', dataset), - ('rows', rows), - ) + tuple(sorted(kwargs.items())) - base_url = endpoint.format(scheme=scheme, netloc=netloc, path=path) + '?' + urlencode(params) - def _extract_ods(): - nonlocal base_url, rows - start = 0 +def path_str(path): + return path if path.startswith('/') else '/' + path + + +@contextual +class OpenDataSoftAPI(Configurable): + dataset = Option(str, required=True) + endpoint = Option(str, default='{scheme}://{netloc}{path}') + scheme = Option(str, default='https') + netloc = Option(str, default='data.opendatasoft.com') + path = Option(path_str, default='/api/records/1.0/search/') + rows = Option(int, default=100) + kwargs = Option(dict, default=dict) + + @ContextProcessor + def compute_path(self, context): + params = (('dataset', self.dataset), ('rows', self.rows), ) + tuple(sorted(self.kwargs.items())) + yield self.endpoint.format(scheme=self.scheme, netloc=self.netloc, path=self.path) + '?' + urlencode(params) + + @ContextProcessor + def start(self, context, base_url): + yield ValueHolder(0) + + def __call__(self, base_url, start, *args, **kwargs): while True: - resp = requests.get('{}&start={start}'.format(base_url, start=start)) + url = '{}&start={start}'.format(base_url, start=start.value) + resp = requests.get(url) records = resp.json().get('records', []) if not len(records): @@ -32,7 +43,14 @@ def from_opendatasoft_api( for row in records: yield {**row.get('fields', {}), 'geometry': row.get('geometry', {})} - start += rows + start.value += self.rows - _extract_ods.__name__ = 'extract_' + dataset.replace('-', '_') - return _extract_ods + +@deprecated +def from_opendatasoft_api(dataset, **kwargs): + return OpenDataSoftAPI(dataset=dataset, **kwargs) + + +__all__ = [ + 'OpenDataSoftAPI', +] diff --git a/bonobo/io/csv.py b/bonobo/io/csv.py index 175737f..73856b7 100644 --- a/bonobo/io/csv.py +++ b/bonobo/io/csv.py @@ -55,10 +55,7 @@ class CsvReader(CsvHandler, FileReader): for row in reader: if len(row) != field_count: - raise ValueError('Got a line with %d fields, expecting %d.' % ( - len(row), - field_count, - )) + raise ValueError('Got a line with %d fields, expecting %d.' % (len(row), field_count, )) yield dict(zip(headers.value, row)) diff --git a/bonobo/io/file.py b/bonobo/io/file.py index 2fd9997..8f6ae8c 100644 --- a/bonobo/io/file.py +++ b/bonobo/io/file.py @@ -1,6 +1,9 @@ +from io import BytesIO + from bonobo.config import Configurable, Option from bonobo.context import ContextProcessor from bonobo.context.processors import contextual +from bonobo.util.file import create_reader from bonobo.util.objects import ValueHolder __all__ = [ @@ -22,8 +25,13 @@ class FileHandler(Configurable): @ContextProcessor def file(self, context): - with self.open() as file: - yield file + if self.path.find('http://') == 0 or self.path.find('https://') == 0: + import requests + response = requests.get(self.path) + yield BytesIO(response.content) + else: + with self.open() as file: + yield file def open(self): return open(self.path, self.mode) diff --git a/bonobo/io/json.py b/bonobo/io/json.py index 04a3a0a..e9d34f1 100644 --- a/bonobo/io/json.py +++ b/bonobo/io/json.py @@ -3,7 +3,9 @@ import json from bonobo.context import ContextProcessor, contextual from .file import FileWriter, FileReader -__all__ = ['JsonWriter', ] +__all__ = [ + 'JsonWriter', +] class JsonHandler: @@ -11,8 +13,10 @@ class JsonHandler: class JsonReader(JsonHandler, FileReader): + loader = staticmethod(json.load) + def read(self, file): - for line in json.load(file): + for line in self.loader(file): yield line diff --git a/bonobo/util/__init__.py b/bonobo/util/__init__.py index 5a2ee26..1b616e1 100644 --- a/bonobo/util/__init__.py +++ b/bonobo/util/__init__.py @@ -65,9 +65,8 @@ def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True if print_values: for k in sorted(row) if sort else row: print( - ' • {t.blue}{k}{t.normal} : {t.black}({tp}){t.normal} {v}{t.clear_eol}'.format( - k=k, v=repr(row[k]), t=term, tp=type(row[k]).__name__ - ) + ' • {t.blue}{k}{t.normal} : {t.black}({tp}){t.normal} {v}{t.clear_eol}'. + format(k=k, v=repr(row[k]), t=term, tp=type(row[k]).__name__) ) yield NOT_MODIFIED diff --git a/bonobo/util/compat.py b/bonobo/util/compat.py index 8a7bb11..8777f09 100644 --- a/bonobo/util/compat.py +++ b/bonobo/util/compat.py @@ -1,6 +1,8 @@ +import functools import struct import sys +import warnings def is_platform_little_endian(): @@ -22,3 +24,20 @@ def is_platform_mac(): def is_platform_32bit(): return struct.calcsize("P") * 8 < 64 + + +def deprecated(func): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emmitted + when the function is used.""" + + @functools.wraps(func) + def new_func(*args, **kwargs): + warnings.simplefilter('always', DeprecationWarning) # turn off filter + warnings.warn( + "Call to deprecated function {}.".format(func.__name__), category=DeprecationWarning, stacklevel=2 + ) + warnings.simplefilter('default', DeprecationWarning) # reset filter + return func(*args, **kwargs) + + return new_func diff --git a/bonobo/util/objects.py b/bonobo/util/objects.py index f6d0a93..f913d5e 100644 --- a/bonobo/util/objects.py +++ b/bonobo/util/objects.py @@ -19,4 +19,4 @@ class Wrapper: class ValueHolder: def __init__(self, value, *, type=None): self.value = value - self.type = type \ No newline at end of file + self.type = type diff --git a/docs/conf.py b/docs/conf.py index c7970e5..e164f8d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -140,7 +140,9 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -latex_documents = [(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'), ] +latex_documents = [ + (master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'), +] # -- Options for manual page output --------------------------------------- @@ -181,7 +183,4 @@ epub_copyright = copyright epub_exclude_files = ['search.html'] # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None) -} - +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} diff --git a/examples b/examples deleted file mode 120000 index 7a35da1..0000000 --- a/examples +++ /dev/null @@ -1 +0,0 @@ -bonobo/examples \ No newline at end of file diff --git a/setup.py b/setup.py index 2e923d8..a9b983c 100644 --- a/setup.py +++ b/setup.py @@ -34,37 +34,35 @@ setup( description='Bonobo', license='Apache License, Version 2.0', install_requires=[ - 'blessings >=1.6,<1.7', 'psutil >=5.0,<5.1', 'requests >=2.12,<2.13', - 'stevedore >=1.19,<1.20', 'toolz >=0.8,<0.9' + 'blessings >=1.6,<1.7', 'psutil >=5.0,<5.1', 'requests >=2.12,<2.13', 'stevedore >=1.19,<1.20', + 'toolz >=0.8,<0.9' ], version=version, long_description=read('README.rst'), classifiers=read('classifiers.txt', tolines), packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, - data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/ext/jupyter/static/extension.js', - 'bonobo/ext/jupyter/static/index.js', - 'bonobo/ext/jupyter/static/index.js.map' - ])], + data_files=[ + ( + 'share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/ext/jupyter/static/extension.js', 'bonobo/ext/jupyter/static/index.js', + 'bonobo/ext/jupyter/static/index.js.map' + ] + ) + ], extras_require={ 'dev': [ - 'coverage >=4.3,<4.4', 'mock >=2.0,<2.1', 'nose >=1.3,<1.4', - 'pylint >=1.6,<1.7', 'pytest >=3,<4', 'pytest-cov >=2.4,<2.5', - 'pytest-timeout >=1.2,<1.3', 'sphinx', 'sphinx_rtd_theme', 'yapf' + 'coverage >=4.3,<4.4', 'mock >=2.0,<2.1', 'nose >=1.3,<1.4', 'pylint >=1.6,<1.7', 'pytest >=3,<4', + 'pytest-cov >=2.4,<2.5', 'pytest-timeout >=1.2,<1.3', 'sphinx', 'sphinx_rtd_theme', 'yapf' ], 'jupyter': ['jupyter >=1.0,<1.1', 'ipywidgets >=6.0.0.beta5'] }, entry_points={ - 'bonobo.commands': [ - 'init = bonobo.commands.init:register', - 'run = bonobo.commands.run:register' - ], + 'bonobo.commands': ['init = bonobo.commands.init:register', 'run = bonobo.commands.run:register'], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], - 'edgy.project.features': - ['bonobo = ' - 'bonobo.ext.edgy.project.feature:BonoboFeature'] + 'edgy.project.features': ['bonobo = ' + 'bonobo.ext.edgy.project.feature:BonoboFeature'] }, url='https://bonobo-project.org/', - download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'. - format(version=version), ) + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), +) diff --git a/tests/core/test_bags.py b/tests/core/test_bags.py index 6c7bb62..b85997a 100644 --- a/tests/core/test_bags.py +++ b/tests/core/test_bags.py @@ -3,10 +3,7 @@ from mock import Mock from bonobo import Bag from bonobo.core.bags import INHERIT_INPUT -args = ( - 'foo', - 'bar', -) +args = ('foo', 'bar', ) kwargs = dict(acme='corp') @@ -39,17 +36,11 @@ def test_inherit(): assert bag.kwargs == {'a': 1} assert bag.flags is () - assert bag2.args == ( - 'a', - 'b', - ) + assert bag2.args == ('a', 'b', ) assert bag2.kwargs == {'a': 1, 'b': 2} assert INHERIT_INPUT in bag2.flags - assert bag3.args == ( - 'a', - 'c', - ) + assert bag3.args == ('a', 'c', ) assert bag3.kwargs == {'a': 1, 'c': 3} assert bag3.flags is () @@ -58,19 +49,12 @@ def test_inherit(): assert bag4.flags is () bag4.set_parent(bag) - assert bag4.args == ( - 'a', - 'd', - ) + assert bag4.args == ('a', 'd', ) assert bag4.kwargs == {'a': 1, 'd': 4} assert bag4.flags is () bag4.set_parent(bag3) - assert bag4.args == ( - 'a', - 'c', - 'd', - ) + assert bag4.args == ('a', 'c', 'd', ) assert bag4.kwargs == {'a': 1, 'c': 3, 'd': 4} assert bag4.flags is () diff --git a/tests/core/test_statistics.py b/tests/core/test_statistics.py index d71e0dc..9b05175 100644 --- a/tests/core/test_statistics.py +++ b/tests/core/test_statistics.py @@ -3,10 +3,7 @@ from bonobo.core.statistics import WithStatistics class MyThingWithStats(WithStatistics): def get_statistics(self, *args, **kwargs): - return ( - ('foo', 42), - ('bar', 69), - ) + return (('foo', 42), ('bar', 69), ) def test_with_statistics(): diff --git a/tests/ext/test_ods.py b/tests/ext/test_ods.py index 79786f2..85036f8 100644 --- a/tests/ext/test_ods.py +++ b/tests/ext/test_ods.py @@ -1,6 +1,7 @@ from mock import patch -from bonobo.ext.opendatasoft import from_opendatasoft_api +from bonobo.ext.opendatasoft import OpenDataSoftAPI +from bonobo.util.objects import ValueHolder class ResponseMock: @@ -13,11 +14,13 @@ class ResponseMock: return {} else: self.count += 1 - return {'records': self.json_value, } + return { + 'records': self.json_value, + } def test_read_from_opendatasoft_api(): - extract = from_opendatasoft_api('http://example.com/', 'test-a-set') + extract = OpenDataSoftAPI(dataset='test-a-set') with patch( 'requests.get', return_value=ResponseMock([ { @@ -32,5 +35,5 @@ def test_read_from_opendatasoft_api(): }, ]) ): - for line in extract(): + for line in extract('http://example.com/', ValueHolder(0)): assert 'foo' in line diff --git a/tests/test_basicusage.py b/tests/test_basicusage.py index 5db6f9d..8889687 100644 --- a/tests/test_basicusage.py +++ b/tests/test_basicusage.py @@ -5,9 +5,7 @@ import bonobo as bb @pytest.mark.timeout(2) def test_run_graph_noop(): - graph = bb.Graph( - bb.noop - ) + graph = bb.Graph(bb.noop) assert len(graph) == 1 result = bb.run(graph, strategy='threadpool') diff --git a/tests/test_commands.py b/tests/test_commands.py index eb6a96a..8c33c6d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -14,14 +14,17 @@ def test_entrypoint(): assert 'init' in commands assert 'run' in commands + def test_no_command(capsys): with pytest.raises(SystemExit): entrypoint([]) out, err = capsys.readouterr() assert 'error: the following arguments are required: command' in err + def test_init(): - pass # need ext dir + pass # need ext dir + def test_run(capsys): entrypoint(['run', '--quiet', get_examples_path('types/strings.py')])