From 89dda0dca6a947f7c43bebfb6bbf68837d4ecad3 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sun, 29 Jul 2018 18:21:56 +0100 Subject: [PATCH] feat: new alternate syntax and switch to black + isort (yeah, maybe not the best time, but that is done). --- bonobo/__init__.py | 51 ++- bonobo/__main__.py | 2 +- bonobo/_api.py | 27 +- bonobo/_version.py | 2 +- bonobo/commands/__init__.py | 15 +- bonobo/commands/base.py | 16 +- bonobo/commands/convert.py | 99 +++-- bonobo/commands/download.py | 16 +- bonobo/commands/examples.py | 20 +- bonobo/commands/init.py | 42 +- bonobo/commands/inspect.py | 6 +- bonobo/commands/run.py | 16 +- bonobo/commands/templates/bare.py-tpl | 1 + bonobo/commands/templates/default.py-tpl | 2 +- bonobo/commands/version.py | 27 +- bonobo/config/__init__.py | 32 +- bonobo/config/configurables.py | 37 +- bonobo/config/functools.py | 4 +- bonobo/config/options.py | 33 +- bonobo/config/processors.py | 38 +- bonobo/config/services.py | 17 +- bonobo/constants.py | 4 +- bonobo/contrib/django/__init__.py | 5 +- bonobo/contrib/django/commands.py | 16 +- bonobo/contrib/google/__init__.py | 26 +- bonobo/contrib/jupyter/__init__.py | 6 +- bonobo/contrib/jupyter/widget.py | 10 +- bonobo/contrib/opendatasoft/__init__.py | 30 +- bonobo/errors.py | 10 +- bonobo/examples/__init__.py | 60 +-- bonobo/examples/__main__.py | 4 +- bonobo/examples/clock.py | 7 +- bonobo/examples/datasets/__main__.py | 37 +- bonobo/examples/datasets/coffeeshops.py | 54 +-- bonobo/examples/datasets/fablabs.py | 20 +- bonobo/examples/empty.py | 2 +- bonobo/examples/environ.py | 2 +- bonobo/examples/files/csv_handlers.py | 10 +- bonobo/examples/files/json_handlers.py | 15 +- bonobo/examples/files/pickle_handlers.py | 23 +- bonobo/examples/files/services.py | 5 +- bonobo/examples/files/text_handlers.py | 10 +- bonobo/examples/types/__main__.py | 2 +- bonobo/examples/types/strings.py | 10 +- bonobo/execution/contexts/__init__.py | 6 +- bonobo/execution/contexts/base.py | 32 +- bonobo/execution/contexts/graph.py | 2 +- bonobo/execution/contexts/node.py | 81 ++-- bonobo/execution/events.py | 12 +- bonobo/execution/strategies/__init__.py | 25 +- bonobo/execution/strategies/base.py | 3 +- bonobo/execution/strategies/executor.py | 10 +- bonobo/nodes/__init__.py | 2 +- bonobo/nodes/basics.py | 79 ++-- bonobo/nodes/io/__init__.py | 20 +- bonobo/nodes/io/base.py | 37 +- bonobo/nodes/io/csv.py | 34 +- bonobo/nodes/io/file.py | 30 +- bonobo/nodes/io/json.py | 14 +- bonobo/nodes/io/pickle.py | 8 +- bonobo/plugins/console.py | 65 ++-- bonobo/plugins/jupyter.py | 4 +- bonobo/registry.py | 30 +- bonobo/settings.py | 31 +- bonobo/structs/graphs.py | 59 ++- bonobo/structs/inputs.py | 6 +- bonobo/structs/tokens.py | 2 +- bonobo/util/__init__.py | 36 +- bonobo/util/api.py | 14 +- bonobo/util/bags.py | 59 +-- bonobo/util/collections.py | 4 +- bonobo/util/compat.py | 10 +- bonobo/util/envelopes.py | 4 +- bonobo/util/environ.py | 33 +- bonobo/util/errors.py | 34 +- bonobo/util/inspect.py | 28 +- bonobo/util/objects.py | 6 +- bonobo/util/pkgs.py | 2 +- bonobo/util/resolvers.py | 8 +- bonobo/util/statistics.py | 6 +- bonobo/util/term.py | 4 +- bonobo/util/testing.py | 66 ++-- tests/commands/test_clibasics.py | 12 +- tests/commands/test_convert.py | 8 +- tests/commands/test_download.py | 13 +- tests/commands/test_init.py | 16 +- tests/commands/test_run.py | 46 +-- tests/commands/test_run_environ.py | 117 +++--- tests/commands/test_version.py | 14 +- tests/config/test_configurables.py | 54 +-- tests/config/test_methods.py | 37 +- tests/config/test_methods_partial.py | 15 +- tests/config/test_processors.py | 18 +- tests/config/test_services.py | 90 +++-- .../test_example_change_some_fields.py | 38 +- .../contexts/test_execution_contexts_graph.py | 6 +- .../contexts/test_execution_contexts_node.py | 102 ++--- tests/execution/test_events.py | 4 +- tests/ext/test_ods.py | 15 +- tests/features/test_inherit.py | 15 +- tests/features/test_not_modified.py | 5 +- tests/nodes/io/test_csv.py | 91 ++--- tests/nodes/io/test_file.py | 19 +- tests/nodes/io/test_json.py | 126 ++---- tests/nodes/io/test_pickle.py | 19 +- tests/nodes/test_basics.py | 31 +- tests/nodes/test_casts.py | 29 +- tests/plugins/test_console.py | 3 +- tests/structs/test_graphs.py | 18 +- tests/structs/test_inputs.py | 18 +- tests/structs/test_tokens.py | 4 +- tests/test_basicusage.py | 2 +- tests/test_execution.py | 2 +- tests/test_publicapi.py | 4 +- tests/test_registry.py | 10 +- tests/test_settings.py | 40 +- tests/util/requireable/dummy.py | 2 +- tests/util/test_bags.py | 358 +++++++++++++----- tests/util/test_collections.py | 18 +- tests/util/test_compat.py | 2 +- tests/util/test_objects.py | 95 ++--- tests/util/test_resolvers.py | 12 +- tests/util/test_statistics.py | 7 +- 123 files changed, 1672 insertions(+), 1640 deletions(-) diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 2a9515f..d605183 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -7,17 +7,50 @@ import sys -assert (sys.version_info >= (3, 5)), 'Python 3.5+ is required to use Bonobo.' +assert sys.version_info >= (3, 5), "Python 3.5+ is required to use Bonobo." from bonobo._api import ( - run, inspect, Graph, create_strategy, open_fs, CsvReader, CsvWriter, FileReader, FileWriter, Filter, FixedWindow, - Format, JsonReader, JsonWriter, LdjsonReader, LdjsonWriter, Limit, OrderFields, PickleReader, PickleWriter, - PrettyPrinter, RateLimited, Rename, SetFields, Tee, UnpackItems, count, identity, noop, create_reader, - create_writer, get_examples_path, open_examples_fs, get_argument_parser, parse_args, __all__, __doc__ + run, + inspect, + Graph, + create_strategy, + open_fs, + CsvReader, + CsvWriter, + FileReader, + FileWriter, + Filter, + FixedWindow, + Format, + JsonReader, + JsonWriter, + LdjsonReader, + LdjsonWriter, + Limit, + OrderFields, + PickleReader, + PickleWriter, + PrettyPrinter, + RateLimited, + Rename, + SetFields, + Tee, + UnpackItems, + count, + identity, + noop, + create_reader, + create_writer, + get_examples_path, + open_examples_fs, + get_argument_parser, + parse_args, + __all__, + __doc__, ) from bonobo._version import __version__ -__all__ = ['__version__'] + __all__ +__all__ = ["__version__"] + __all__ __logo__ = '' __doc__ = __doc__ __version__ = __version__ @@ -32,10 +65,8 @@ def _repr_html_(): '
' '
{}
' ' 
{}
' - '
' - ).format( - __logo__, '
'.join(get_versions(all=True)) - ) + "" + ).format(__logo__, "
".join(get_versions(all=True))) del sys diff --git a/bonobo/__main__.py b/bonobo/__main__.py index 05ed120..fac20ca 100644 --- a/bonobo/__main__.py +++ b/bonobo/__main__.py @@ -1,4 +1,4 @@ from bonobo.commands import entrypoint -if __name__ == '__main__': +if __name__ == "__main__": entrypoint() diff --git a/bonobo/_api.py b/bonobo/_api.py index f5463c7..78576aa 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -45,14 +45,17 @@ def run(graph, *, plugins=None, services=None, strategy=None): plugins = plugins or [] from bonobo import settings + settings.check() if not settings.QUIET.get(): # pragma: no cover if _is_interactive_console(): import mondrian + mondrian.setup(excepthook=True) from bonobo.plugins.console import ConsoleOutputPlugin + if ConsoleOutputPlugin not in plugins: plugins.append(ConsoleOutputPlugin) @@ -61,20 +64,23 @@ def run(graph, *, plugins=None, services=None, strategy=None): from bonobo.contrib.jupyter import JupyterOutputPlugin except ImportError: import logging + logging.warning( 'Failed to load jupyter widget. Easiest way is to install the optional "jupyter" ' - 'dependencies with «pip install bonobo[jupyter]», but you can also install a specific ' - 'version by yourself.' + "dependencies with «pip install bonobo[jupyter]», but you can also install a specific " + "version by yourself." ) else: if JupyterOutputPlugin not in plugins: plugins.append(JupyterOutputPlugin) import logging + logging.getLogger().setLevel(settings.LOGGING_LEVEL.get()) strategy = create_strategy(strategy) from bonobo.util.errors import sweeten_errors + with sweeten_errors(): return strategy.execute(graph, plugins=plugins, services=services) @@ -83,15 +89,15 @@ def _inspect_as_graph(graph): return graph._repr_dot_() -_inspect_formats = {'graph': _inspect_as_graph} +_inspect_formats = {"graph": _inspect_as_graph} @api.register_graph def inspect(graph, *, plugins=None, services=None, strategy=None, format): if not format in _inspect_formats: raise NotImplementedError( - 'Output format {} not implemented. Choices are: {}.'.format( - format, ', '.join(sorted(_inspect_formats.keys())) + "Output format {} not implemented. Choices are: {}.".format( + format, ", ".join(sorted(_inspect_formats.keys())) ) ) print(_inspect_formats[format](graph)) @@ -160,20 +166,18 @@ api.register_group( ) # registry -api.register_group( - create_reader, - create_writer, -) +api.register_group(create_reader, create_writer) def _is_interactive_console(): import sys + return sys.stdout.isatty() def _is_jupyter_notebook(): try: - return get_ipython().__class__.__name__ == 'ZMQInteractiveShell' + return get_ipython().__class__.__name__ == "ZMQInteractiveShell" except NameError: return False @@ -182,7 +186,8 @@ def _is_jupyter_notebook(): def get_examples_path(*pathsegments): import os import pathlib - return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments)) + + return str(pathlib.Path(os.path.dirname(__file__), "examples", *pathsegments)) @api.register diff --git a/bonobo/_version.py b/bonobo/_version.py index aece342..22049ab 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.6.2' +__version__ = "0.6.2" diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index a482b53..e1c0d2d 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -19,9 +19,9 @@ def entrypoint(args=None): logger.setLevel(settings.LOGGING_LEVEL.get()) parser = argparse.ArgumentParser() - parser.add_argument('--debug', '-D', action='store_true') + parser.add_argument("--debug", "-D", action="store_true") - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") subparsers.required = True commands = {} @@ -39,23 +39,24 @@ def entrypoint(args=None): # old school, function based. commands[ext.name] = ext.plugin(parser) except Exception: - logger.exception('Error while loading command {}.'.format(ext.name)) + logger.exception("Error while loading command {}.".format(ext.name)) from stevedore import ExtensionManager - mgr = ExtensionManager(namespace='bonobo.commands') + + mgr = ExtensionManager(namespace="bonobo.commands") mgr.map(register_extension) parsed_args = parser.parse_args(args).__dict__ - if parsed_args.pop('debug', False): + if parsed_args.pop("debug", False): settings.DEBUG.set(True) settings.LOGGING_LEVEL.set(logging.DEBUG) logger.setLevel(settings.LOGGING_LEVEL.get()) - logger.debug('Command: ' + parsed_args['command'] + ' Arguments: ' + repr(parsed_args)) + logger.debug("Command: " + parsed_args["command"] + " Arguments: " + repr(parsed_args)) # Get command handler, execute, rince. - command = commands[parsed_args.pop('command')] + command = commands[parsed_args.pop("command")] command(**parsed_args) return 0 diff --git a/bonobo/commands/base.py b/bonobo/commands/base.py index 4f15daa..54ec3c6 100644 --- a/bonobo/commands/base.py +++ b/bonobo/commands/base.py @@ -33,7 +33,7 @@ class BaseCommand: """ The actual logic of the command. Subclasses must implement this method. """ - raise NotImplementedError('Subclasses of BaseCommand must provide a handle() method') + raise NotImplementedError("Subclasses of BaseCommand must provide a handle() method") class BaseGraphCommand(BaseCommand): @@ -48,8 +48,8 @@ class BaseGraphCommand(BaseCommand): def add_arguments(self, parser): # target arguments (cannot provide both). source_group = parser.add_mutually_exclusive_group(required=self.required) - source_group.add_argument('file', nargs='?', type=str) - source_group.add_argument('-m', dest='mod', type=str) + source_group.add_argument("file", nargs="?", type=str) + source_group.add_argument("-m", dest="mod", type=str) # add arguments to enforce system environment. parser = get_argument_parser(parser) @@ -66,7 +66,7 @@ class BaseGraphCommand(BaseCommand): def do_handle(self, graph, **options): if not self.handler: - raise RuntimeError('{} has no handler defined.'.format(get_name(self))) + raise RuntimeError("{} has no handler defined.".format(get_name(self))) return self.handler(graph, **options) @contextmanager @@ -87,20 +87,20 @@ class BaseGraphCommand(BaseCommand): sys.argv = [mod] self._run_module(mod) else: - raise RuntimeError('No target provided.') + raise RuntimeError("No target provided.") finally: sys.argv = _argv if _graph is None: - raise RuntimeError('Could not find graph.') + raise RuntimeError("Could not find graph.") yield _graph, _graph_execution_options, options def _run_path(self, file): - return runpy.run_path(file, run_name='__main__') + return runpy.run_path(file, run_name="__main__") def _run_module(self, mod): - return runpy.run_module(mod, run_name='__main__') + return runpy.run_module(mod, run_name="__main__") @contextmanager diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index 91d540c..c94d37b 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -6,82 +6,75 @@ from bonobo.util.resolvers import _resolve_options, _resolve_transformations class ConvertCommand(BaseCommand): def add_arguments(self, parser): - parser.add_argument('input_filename', help='Input filename.') - parser.add_argument('output_filename', help='Output filename.') + parser.add_argument("input_filename", help="Input filename.") + parser.add_argument("output_filename", help="Output filename.") parser.add_argument( - '--' + READER, - '-r', - help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.' + "--" + READER, + "-r", + help="Choose the reader factory if it cannot be detected from extension, or if detection is wrong.", ) parser.add_argument( - '--' + WRITER, - '-w', - help= - 'Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).' + "--" + WRITER, + "-w", + help="Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).", + ) + parser.add_argument("--limit", "-l", type=int, help="Adds a Limit() after the reader instance.", default=None) + parser.add_argument( + "--transformation", + "-t", + dest="transformation", + action="append", + help="Add a transformation between input and output (can be used multiple times, order is preserved).", ) parser.add_argument( - '--limit', - '-l', - type=int, - help='Adds a Limit() after the reader instance.', - default=None, - ) - parser.add_argument( - '--transformation', - '-t', - dest='transformation', - action='append', - help='Add a transformation between input and output (can be used multiple times, order is preserved).', - ) - parser.add_argument( - '--option', - '-O', - dest='option', - action='append', + "--option", + "-O", + dest="option", + action="append", help='Add a named option to both reader and writer factories (i.e. foo="bar").', ) parser.add_argument( - '--' + READER + '-option', - '-' + READER[0].upper(), - dest=READER + '_option', - action='append', - help='Add a named option to the reader factory.', + "--" + READER + "-option", + "-" + READER[0].upper(), + dest=READER + "_option", + action="append", + help="Add a named option to the reader factory.", ) parser.add_argument( - '--' + WRITER + '-option', - '-' + WRITER[0].upper(), - dest=WRITER + '_option', - action='append', - help='Add a named option to the writer factory.', + "--" + WRITER + "-option", + "-" + WRITER[0].upper(), + dest=WRITER + "_option", + action="append", + help="Add a named option to the writer factory.", ) def handle( - self, - input_filename, - output_filename, - reader=None, - reader_option=None, - writer=None, - writer_option=None, - option=None, - limit=None, - transformation=None, + self, + input_filename, + output_filename, + reader=None, + reader_option=None, + writer=None, + writer_option=None, + option=None, + limit=None, + transformation=None, ): reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader) reader_kwargs = _resolve_options((option or []) + (reader_option or [])) - if output_filename == '-': + if output_filename == "-": writer_factory = bonobo.PrettyPrinter writer_args = () else: writer_factory = default_registry.get_writer_factory_for(output_filename, format=writer) - writer_args = (output_filename, ) + writer_args = (output_filename,) writer_kwargs = _resolve_options((option or []) + (writer_option or [])) transformations = () if limit: - transformations += (bonobo.Limit(limit), ) + transformations += (bonobo.Limit(limit),) transformations += _resolve_transformations(transformation) @@ -92,8 +85,4 @@ class ConvertCommand(BaseCommand): writer_factory(*writer_args, **writer_kwargs), ) - return bonobo.run( - graph, services={ - 'fs': bonobo.open_fs(), - } - ) + return bonobo.run(graph, services={"fs": bonobo.open_fs()}) diff --git a/bonobo/commands/download.py b/bonobo/commands/download.py index 96b1c2f..ae3a1a3 100644 --- a/bonobo/commands/download.py +++ b/bonobo/commands/download.py @@ -6,28 +6,28 @@ import requests import bonobo from bonobo.commands import BaseCommand -EXAMPLES_BASE_URL = 'https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/' +EXAMPLES_BASE_URL = "https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/" """The URL to our git repository, in raw mode.""" class DownloadCommand(BaseCommand): def handle(self, *, path, **options): - if not path.startswith('examples'): - raise ValueError('Download command currently supports examples only') - examples_path = re.sub('^examples/', '', path) + if not path.startswith("examples"): + raise ValueError("Download command currently supports examples only") + examples_path = re.sub("^examples/", "", path) output_path = bonobo.get_examples_path(examples_path) - with _open_url(EXAMPLES_BASE_URL + examples_path) as response, open(output_path, 'wb') as fout: + with _open_url(EXAMPLES_BASE_URL + examples_path) as response, open(output_path, "wb") as fout: for chunk in response.iter_content(io.DEFAULT_BUFFER_SIZE): fout.write(chunk) - self.logger.info('Download saved to {}'.format(output_path)) + self.logger.info("Download saved to {}".format(output_path)) def add_arguments(self, parser): - parser.add_argument('path', help='The relative path of the thing to download.') + parser.add_argument("path", help="The relative path of the thing to download.") def _open_url(url): """Open a HTTP connection to the URL and return a file-like object.""" response = requests.get(url, stream=True) if response.status_code != 200: - raise IOError('Unable to download {}, HTTP {}'.format(url, response.status_code)) + raise IOError("Unable to download {}, HTTP {}".format(url, response.status_code)) return response diff --git a/bonobo/commands/examples.py b/bonobo/commands/examples.py index e6a2764..835c027 100644 --- a/bonobo/commands/examples.py +++ b/bonobo/commands/examples.py @@ -1,23 +1,23 @@ from bonobo.commands import BaseCommand all_examples = ( - 'clock', - 'datasets', - 'environ', - 'files.csv_handlers', - 'files.json_handlers', - 'files.pickle_handlers', - 'files.text_handlers', - 'types', + "clock", + "datasets", + "environ", + "files.csv_handlers", + "files.json_handlers", + "files.pickle_handlers", + "files.text_handlers", + "types", ) class ExamplesCommand(BaseCommand): def handle(self): - print('You can run the following examples:') + print("You can run the following examples:") print() for example in all_examples: - print(' $ python -m bonobo.examples.{}'.format(example)) + print(" $ python -m bonobo.examples.{}".format(example)) print() def add_arguments(self, parser): diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index 6d4b217..7713912 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -6,67 +6,67 @@ from bonobo.commands import BaseCommand class InitCommand(BaseCommand): - TEMPLATES = {'bare', 'default'} - TEMPLATES_PATH = os.path.join(os.path.dirname(__file__), 'templates') + TEMPLATES = {"bare", "default"} + TEMPLATES_PATH = os.path.join(os.path.dirname(__file__), "templates") def add_arguments(self, parser): - parser.add_argument('filename') - parser.add_argument('--force', '-f', default=False, action='store_true') + parser.add_argument("filename") + parser.add_argument("--force", "-f", default=False, action="store_true") target_group = parser.add_mutually_exclusive_group(required=False) - target_group.add_argument('--template', '-t', choices=self.TEMPLATES, default='default') - target_group.add_argument('--package', '-p', action='store_true', default=False) + target_group.add_argument("--template", "-t", choices=self.TEMPLATES, default="default") + target_group.add_argument("--package", "-p", action="store_true", default=False) def create_file_from_template(self, *, template, filename): template_name = template name, ext = os.path.splitext(filename) - if ext != '.py': + if ext != ".py": raise ValueError('Filenames should end with ".py".') loader = FileSystemLoader(self.TEMPLATES_PATH) env = Environment(loader=loader) - template = env.get_template(template_name + '.py-tpl') + template = env.get_template(template_name + ".py-tpl") - with open(filename, 'w+') as f: + with open(filename, "w+") as f: f.write(template.render(name=name)) - self.logger.info('Generated {} using template {!r}.'.format(filename, template_name)) + self.logger.info("Generated {} using template {!r}.".format(filename, template_name)) def create_package(self, *, filename): name, ext = os.path.splitext(filename) - if ext != '': - raise ValueError('Package names should not have an extension.') + if ext != "": + raise ValueError("Package names should not have an extension.") try: import medikit.commands except ImportError as exc: raise ImportError( - 'To initialize a package, you need to install medikit (pip install --upgrade medikit).' + "To initialize a package, you need to install medikit (pip install --upgrade medikit)." ) from exc package_name = os.path.basename(filename) medikit.commands.handle_init( - os.path.join(os.getcwd(), filename, 'Projectfile'), name=package_name, requirements=['bonobo'] + os.path.join(os.getcwd(), filename, "Projectfile"), name=package_name, requirements=["bonobo"] ) self.logger.info('Generated "{}" package with medikit.'.format(package_name)) - self.create_file_from_template(template='default', filename=os.path.join(filename, package_name, '__main__.py')) + self.create_file_from_template(template="default", filename=os.path.join(filename, package_name, "__main__.py")) print('Your "{}" package has been created.'.format(package_name)) print() - print('Install it...') + print("Install it...") print() - print(' pip install --editable {}'.format(filename)) + print(" pip install --editable {}".format(filename)) print() - print('Then maybe run the example...') + print("Then maybe run the example...") print() - print(' python -m {}'.format(package_name)) + print(" python -m {}".format(package_name)) print() - print('Enjoy!') + print("Enjoy!") def handle(self, *, template, filename, package=False, force=False): if os.path.exists(filename) and not force: - raise FileExistsError('Target filename already exists, use --force to override.') + raise FileExistsError("Target filename already exists, use --force to override.") if package: self.create_package(filename=filename) diff --git a/bonobo/commands/inspect.py b/bonobo/commands/inspect.py index 0e6dcd4..31e975f 100644 --- a/bonobo/commands/inspect.py +++ b/bonobo/commands/inspect.py @@ -7,9 +7,9 @@ class InspectCommand(BaseGraphCommand): def add_arguments(self, parser): super(InspectCommand, self).add_arguments(parser) - parser.add_argument('--graph', '-g', dest='format', action='store_const', const='graph') + parser.add_argument("--graph", "-g", dest="format", action="store_const", const="graph") def parse_options(self, **options): - if not options.get('format'): - raise RuntimeError('You must provide a format (try --graph).') + if not options.get("format"): + raise RuntimeError("You must provide a format (try --graph).") return options diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index ce76bfc..4dee7cc 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -12,13 +12,14 @@ class RunCommand(BaseGraphCommand): super(RunCommand, self).add_arguments(parser) verbosity_group = parser.add_mutually_exclusive_group() - verbosity_group.add_argument('--quiet', '-q', action='store_true') - verbosity_group.add_argument('--verbose', '-v', action='store_true') + verbosity_group.add_argument("--quiet", "-q", action="store_true") + verbosity_group.add_argument("--verbose", "-v", action="store_true") - parser.add_argument('--install', '-I', action='store_true') + parser.add_argument("--install", "-I", action="store_true") def parse_options(self, *, quiet=False, verbose=False, install=False, **options): from bonobo import settings + settings.QUIET.set_if_true(quiet) settings.DEBUG.set_if_true(verbose) self.install = install @@ -28,9 +29,9 @@ class RunCommand(BaseGraphCommand): # add install logic if self.install: if os.path.isdir(file): - requirements = os.path.join(file, 'requirements.txt') + requirements = os.path.join(file, "requirements.txt") else: - requirements = os.path.join(os.path.dirname(file), 'requirements.txt') + requirements = os.path.join(os.path.dirname(file), "requirements.txt") _install_requirements(requirements) return super()._run_path(file) @@ -38,7 +39,7 @@ class RunCommand(BaseGraphCommand): def _run_module(self, mod): # install not implemented for a module, not sure it even make sense. if self.install: - raise RuntimeError('--install behaviour when running a module is not defined.') + raise RuntimeError("--install behaviour when running a module is not defined.") return super()._run_module(mod) @@ -59,10 +60,11 @@ def _install_requirements(requirements): import importlib import pip - pip.main(['install', '-r', requirements]) + pip.main(["install", "-r", requirements]) # Some shenanigans to be sure everything is importable after this, especially .egg-link files which # are referenced in *.pth files and apparently loaded by site.py at some magic bootstrap moment of the # python interpreter. pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources) import site + importlib.reload(site) diff --git a/bonobo/commands/templates/bare.py-tpl b/bonobo/commands/templates/bare.py-tpl index 1ca3019..9390d76 100644 --- a/bonobo/commands/templates/bare.py-tpl +++ b/bonobo/commands/templates/bare.py-tpl @@ -3,6 +3,7 @@ import bonobo def get_graph(**options): graph = bonobo.Graph() + graph.get_cursor() >> ... return graph diff --git a/bonobo/commands/templates/default.py-tpl b/bonobo/commands/templates/default.py-tpl index eaea053..fef1e1c 100644 --- a/bonobo/commands/templates/default.py-tpl +++ b/bonobo/commands/templates/default.py-tpl @@ -27,7 +27,7 @@ def get_graph(**options): """ graph = bonobo.Graph() - graph.add_chain(extract, transform, load) + graph.get_cursor() >> extract >> transform >> load return graph diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index 5ca2311..70468ef 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -9,15 +9,15 @@ def get_versions(*, all=False, quiet=None): if all: for name in sorted(bonobo_packages): - if name != 'bonobo': + if name != "bonobo": try: - mod = __import__(name.replace('-', '_')) + mod = __import__(name.replace("-", "_")) try: yield _format_version(mod, name=name, quiet=quiet) except Exception as exc: - yield '{} ({})'.format(name, exc) + yield "{} ({})".format(name, exc) except ImportError as exc: - yield '{} is not importable ({}).'.format(name, exc) + yield "{} is not importable ({}).".format(name, exc) class VersionCommand(BaseCommand): @@ -26,23 +26,24 @@ class VersionCommand(BaseCommand): print(line) def add_arguments(self, parser): - parser.add_argument('--all', '-a', action='store_true') - parser.add_argument('--quiet', '-q', action='count') + parser.add_argument("--all", "-a", action="store_true") + parser.add_argument("--quiet", "-q", action="count") def _format_version(mod, *, name=None, quiet=False): from bonobo.util.pkgs import bonobo_packages + args = { - 'name': name or mod.__name__, - 'version': mod.__version__, - 'location': bonobo_packages[name or mod.__name__].location + "name": name or mod.__name__, + "version": mod.__version__, + "location": bonobo_packages[name or mod.__name__].location, } if not quiet: - return '{name} v.{version} (in {location})'.format(**args) + return "{name} v.{version} (in {location})".format(**args) if quiet < 2: - return '{name} {version}'.format(**args) + return "{name} {version}".format(**args) if quiet < 3: - return '{version}'.format(**args) + return "{version}".format(**args) - raise RuntimeError('Hard to be so quiet...') + raise RuntimeError("Hard to be so quiet...") diff --git a/bonobo/config/__init__.py b/bonobo/config/__init__.py index 809fc05..344b608 100644 --- a/bonobo/config/__init__.py +++ b/bonobo/config/__init__.py @@ -11,23 +11,23 @@ from bonobo.config.processors import ContextProcessor, use_context, use_context_ from bonobo.config.services import Container, Exclusive, Service, use, create_container from bonobo.util import deprecated_alias -requires = deprecated_alias('requires', use) +requires = deprecated_alias("requires", use) # Bonobo's Config API __all__ = [ - 'Configurable', - 'Container', - 'ContextProcessor', - 'Exclusive', - 'Method', - 'Option', - 'Service', - 'create_container', - 'requires', - 'transformation_factory', - 'use', - 'use_context', - 'use_context_processor', - 'use_no_input', - 'use_raw_input', + "Configurable", + "Container", + "ContextProcessor", + "Exclusive", + "Method", + "Option", + "Service", + "create_container", + "requires", + "transformation_factory", + "use", + "use_context", + "use_context_processor", + "use_no_input", + "use_raw_input", ] diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index 85a89c9..579b1ad 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -1,9 +1,7 @@ from bonobo.errors import AbstractError from bonobo.util import get_name, iscontextprocessor, isoption, sortedlist -__all__ = [ - 'Configurable', -] +__all__ = ["Configurable"] get_creation_counter = lambda v: v._creation_counter @@ -42,12 +40,12 @@ class ConfigurableMeta(type): for _positional, _counter, _name, _value in cls.__options: _param = _name if _value.type: - _param = get_name(_value.type) + ' ' + _param + _param = get_name(_value.type) + " " + _param - prefix = ':param {}: '.format(_param) - for lineno, line in enumerate((_value.__doc__ or '').split('\n')): - _options_doc.append((' ' * len(prefix) if lineno else prefix) + line) - cls.__doc__ = '\n\n'.join(map(str.strip, filter(None, (cls.__doc__, '\n'.join(_options_doc))))) + prefix = ":param {}: ".format(_param) + for lineno, line in enumerate((_value.__doc__ or "").split("\n")): + _options_doc.append((" " * len(prefix) if lineno else prefix) + line) + cls.__doc__ = "\n\n".join(map(str.strip, filter(None, (cls.__doc__, "\n".join(_options_doc))))) @property def __options__(cls): @@ -64,10 +62,7 @@ class ConfigurableMeta(type): return cls.__processors_cache def __repr__(self): - return ' '.join(( - ' 1 else '', ', '.join(map(repr, sorted(extraneous))) + "{}() got {} unexpected option{}: {}.".format( + cls.__name__, + len(extraneous), + "s" if len(extraneous) > 1 else "", + ", ".join(map(repr, sorted(extraneous))), ) ) @@ -165,9 +162,11 @@ class Configurable(metaclass=ConfigurableMeta): if len(missing): if _final: raise TypeError( - '{}() missing {} required option{}: {}.'.format( - cls.__name__, len(missing), 's' - if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing))) + "{}() missing {} required option{}: {}.".format( + cls.__name__, + len(missing), + "s" if len(missing) > 1 else "", + ", ".join(map(repr, sorted(missing))), ) ) return PartiallyConfigured(cls, *args, **kwargs) @@ -196,7 +195,7 @@ class Configurable(metaclass=ConfigurableMeta): break if name in self._options_values: - raise ValueError('Already got a value for option {}'.format(name)) + raise ValueError("Already got a value for option {}".format(name)) setattr(self, name, args[position]) position += 1 diff --git a/bonobo/config/functools.py b/bonobo/config/functools.py index 4bf9c31..5b5a674 100644 --- a/bonobo/config/functools.py +++ b/bonobo/config/functools.py @@ -6,8 +6,8 @@ def transformation_factory(f): @functools.wraps(f) def _transformation_factory(*args, **kwargs): retval = f(*args, **kwargs) - retval.__name__ = f.__name__ + '({})'.format( - ', '.join(itertools.chain(map(repr, args), ('{}={!r}'.format(k, v) for k, v in kwargs.items()))) + retval.__name__ = f.__name__ + "({})".format( + ", ".join(itertools.chain(map(repr, args), ("{}={!r}".format(k, v) for k, v in kwargs.items()))) ) return retval diff --git a/bonobo/config/options.py b/bonobo/config/options.py index cdf9411..1f432e2 100644 --- a/bonobo/config/options.py +++ b/bonobo/config/options.py @@ -66,9 +66,9 @@ class Option: # Docstring formating self.__doc__ = __doc__ or None if self.__doc__: - self.__doc__ = textwrap.dedent(self.__doc__.strip('\n')).strip() + self.__doc__ = textwrap.dedent(self.__doc__.strip("\n")).strip() if default: - self.__doc__ += '\n\nDefault: {!r}'.format(default) + self.__doc__ += "\n\nDefault: {!r}".format(default) # This hack is necessary for python3.5 self._creation_counter = Option._creation_counter @@ -88,13 +88,13 @@ class Option: inst._options_values[self.name] = self.clean(value) def __repr__(self): - return '<{positional}{typename} {type}{name} default={default!r}{required}>'.format( + return "<{positional}{typename} {type}{name} default={default!r}{required}>".format( typename=type(self).__name__, - type='({})'.format(self.type) if istype(self.type) else '', + type="({})".format(self.type) if istype(self.type) else "", name=self.name, - positional='*' if self.positional else '**', + positional="*" if self.positional else "**", default=self.default, - required=' (required)' if self.required else '', + required=" (required)" if self.required else "", ) def clean(self, value): @@ -106,15 +106,16 @@ class Option: class RemovedOption(Option): def __init__(self, *args, value, **kwargs): - kwargs['required'] = False + kwargs["required"] = False super(RemovedOption, self).__init__(*args, **kwargs) self.value = value def clean(self, value): if value != self.value: raise ValueError( - 'Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.'. - format(self.name, self.value) + "Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.".format( + self.name, self.value + ) ) return self.value @@ -129,12 +130,12 @@ class RenamedOption(Option): def __get__(self, instance, owner): raise ValueError( - 'Trying to get value from renamed option {}, try getting {} instead.'.format(self.name, self.target) + "Trying to get value from renamed option {}, try getting {} instead.".format(self.name, self.target) ) def clean(self, value): raise ValueError( - 'Trying to set value of renamed option {}, try setting {} instead.'.format(self.name, self.target) + "Trying to set value of renamed option {}, try setting {} instead.".format(self.name, self.target) ) @@ -182,7 +183,7 @@ class Method(Option): # If a callable is provided as default, then use self as if it was used as a decorator if default is not None: if not callable(default): - raise ValueError('Method defaults should be callable, if provided.') + raise ValueError("Method defaults should be callable, if provided.") self(default) def __get__(self, inst, type_): @@ -194,17 +195,15 @@ class Method(Option): def __set__(self, inst, value): if not callable(value): raise TypeError( - 'Option {!r} ({}) is expecting a callable value, got {!r} object: {!r}.'.format( - self.name, - type(self).__name__, - type(value).__name__, value + "Option {!r} ({}) is expecting a callable value, got {!r} object: {!r}.".format( + self.name, type(self).__name__, type(value).__name__, value ) ) inst._options_values[self.name] = self.type(value) if self.type else value def __call__(self, impl): if self.default: - raise RuntimeError('Can only be used once as a decorator.') + raise RuntimeError("Can only be used once as a decorator.") self.default = impl self.required = False return self diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index c8a2ddf..ab21a28 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -53,7 +53,7 @@ class ContextProcessor(Option): self.name = self.__name__ def __repr__(self): - return repr(self.func).replace(' start): - url = '{}&start={start}&rows={rows}'.format( + url = "{}&start={start}&rows={rows}".format( base_url, start=start.value, rows=self.rows if not self.limit else min(self.rows, self.limit - start) ) resp = requests.get(url) - records = resp.json().get('records', []) + records = resp.json().get("records", []) if not len(records): break for row in records: - yield { - **row.get('fields', {}), - 'geometry': row.get('geometry', {}), - 'recordid': row.get('recordid'), - } + yield {**row.get("fields", {}), "geometry": row.get("geometry", {}), "recordid": row.get("recordid")} start += self.rows -__all__ = [ - 'OpenDataSoftAPI', -] +__all__ = ["OpenDataSoftAPI"] diff --git a/bonobo/errors.py b/bonobo/errors.py index 173ce40..c0fb3fa 100644 --- a/bonobo/errors.py +++ b/bonobo/errors.py @@ -16,10 +16,7 @@ class InactiveWritableError(InactiveIOError): class ValidationError(RuntimeError): def __init__(self, inst, message): super(ValidationError, self).__init__( - 'Validation error in {class_name}: {message}'.format( - class_name=type(inst).__name__, - message=message, - ) + "Validation error in {class_name}: {message}".format(class_name=type(inst).__name__, message=message) ) @@ -41,9 +38,8 @@ class AbstractError(UnrecoverableError, NotImplementedError): def __init__(self, method): super().__init__( - 'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format( - class_name=get_name(method.__self__), - method_name=get_name(method), + "Call to abstract method {class_name}.{method_name}(...): missing implementation.".format( + class_name=get_name(method.__self__), method_name=get_name(method) ) ) diff --git a/bonobo/examples/__init__.py b/bonobo/examples/__init__.py index eaea6f0..ed8a873 100644 --- a/bonobo/examples/__init__.py +++ b/bonobo/examples/__init__.py @@ -8,40 +8,21 @@ from bonobo.util.statistics import Timer def get_argument_parser(parser=None): parser = bonobo.get_argument_parser(parser=parser) + parser.add_argument("--limit", "-l", type=int, default=None, help="If set, limits the number of processed lines.") parser.add_argument( - '--limit', - '-l', - type=int, - default=None, - help='If set, limits the number of processed lines.' - ) - parser.add_argument( - '--print', - '-p', - action='store_true', - default=False, - help='If set, pretty prints before writing to output file.' + "--print", "-p", action="store_true", default=False, help="If set, pretty prints before writing to output file." ) - parser.add_argument( - '--strategy', - '-s', - type=str, - choices=STRATEGIES.keys(), - default=DEFAULT_STRATEGY, - ) + parser.add_argument("--strategy", "-s", type=str, choices=STRATEGIES.keys(), default=DEFAULT_STRATEGY) return parser def get_graph_options(options): - _limit = options.pop('limit', None) - _print = options.pop('print', False) + _limit = options.pop("limit", None) + _print = options.pop("print", False) - return { - '_limit': (bonobo.Limit(_limit), ) if _limit else (), - '_print': (bonobo.PrettyPrinter(), ) if _print else (), - } + return {"_limit": (bonobo.Limit(_limit),) if _limit else (), "_print": (bonobo.PrettyPrinter(),) if _print else ()} def run(get_graph, get_services, *, parser=None): @@ -49,38 +30,29 @@ def run(get_graph, get_services, *, parser=None): with bonobo.parse_args(parser) as options: with Timer() as timer: - print( - 'Options:', ' '.join( - '{}={}'.format(k, v) - for k, v in sorted(options.items()) - ) - ) + print("Options:", " ".join("{}={}".format(k, v) for k, v in sorted(options.items()))) retval = bonobo.run( - get_graph(**get_graph_options(options)), - services=get_services(), - strategy=options['strategy'], + get_graph(**get_graph_options(options)), services=get_services(), strategy=options["strategy"] ) - print('Execution time:', timer) - print('Return value:', retval) - print('XStatus:', retval.xstatus) + print("Execution time:", timer) + print("Return value:", retval) + print("XStatus:", retval.xstatus) return retval.xstatus def get_minor_version(): - return '.'.join(bonobo.__version__.split('.')[:2]) + return ".".join(bonobo.__version__.split(".")[:2]) def get_datasets_dir(*dirs): - home_dir = os.path.expanduser('~') - target_dir = os.path.join( - home_dir, '.cache/bonobo', get_minor_version(), *dirs - ) + home_dir = os.path.expanduser("~") + target_dir = os.path.join(home_dir, ".cache/bonobo", get_minor_version(), *dirs) os.makedirs(target_dir, exist_ok=True) return target_dir def get_services(): return { - 'fs': bonobo.open_fs(get_datasets_dir('datasets')), - 'fs.static': bonobo.open_examples_fs('datasets', 'static'), + "fs": bonobo.open_fs(get_datasets_dir("datasets")), + "fs.static": bonobo.open_examples_fs("datasets", "static"), } diff --git a/bonobo/examples/__main__.py b/bonobo/examples/__main__.py index 92cc165..fd4eb40 100644 --- a/bonobo/examples/__main__.py +++ b/bonobo/examples/__main__.py @@ -1,5 +1,5 @@ -if __name__ == '__main__': +if __name__ == "__main__": from bonobo.commands import entrypoint import sys - entrypoint(['examples'] + sys.argv[1:]) + entrypoint(["examples"] + sys.argv[1:]) diff --git a/bonobo/examples/clock.py b/bonobo/examples/clock.py index fda1ce7..57c4924 100644 --- a/bonobo/examples/clock.py +++ b/bonobo/examples/clock.py @@ -14,15 +14,12 @@ def extract(): def get_graph(): graph = bonobo.Graph() - graph.add_chain( - extract, - print, - ) + graph.add_chain(extract, print) return graph -if __name__ == '__main__': +if __name__ == "__main__": parser = bonobo.get_argument_parser() with bonobo.parse_args(parser): bonobo.run(get_graph()) diff --git a/bonobo/examples/datasets/__main__.py b/bonobo/examples/datasets/__main__.py index 1419a08..53a8a60 100644 --- a/bonobo/examples/datasets/__main__.py +++ b/bonobo/examples/datasets/__main__.py @@ -6,24 +6,16 @@ from bonobo.examples import get_datasets_dir, get_minor_version, get_services from bonobo.examples.datasets.coffeeshops import get_graph as get_coffeeshops_graph from bonobo.examples.datasets.fablabs import get_graph as get_fablabs_graph -graph_factories = { - 'coffeeshops': get_coffeeshops_graph, - 'fablabs': get_fablabs_graph, -} +graph_factories = {"coffeeshops": get_coffeeshops_graph, "fablabs": get_fablabs_graph} -if __name__ == '__main__': +if __name__ == "__main__": parser = examples.get_argument_parser() - parser.add_argument( - '--target', '-t', choices=graph_factories.keys(), nargs='+' - ) - parser.add_argument('--sync', action='store_true', default=False) + parser.add_argument("--target", "-t", choices=graph_factories.keys(), nargs="+") + parser.add_argument("--sync", action="store_true", default=False) with bonobo.parse_args(parser) as options: graph_options = examples.get_graph_options(options) - graph_names = list( - options['target'] - if options['target'] else sorted(graph_factories.keys()) - ) + graph_names = list(options["target"] if options["target"] else sorted(graph_factories.keys())) # Create a graph with all requested subgraphs graph = bonobo.Graph() @@ -32,29 +24,20 @@ if __name__ == '__main__': bonobo.run(graph, services=get_services()) - if options['sync']: + if options["sync"]: # TODO: when parallel option for node will be implemented, need to be rewriten to use a graph. import boto3 - s3 = boto3.client('s3') + s3 = boto3.client("s3") local_dir = get_datasets_dir() for root, dirs, files in os.walk(local_dir): for filename in files: local_path = os.path.join(root, filename) relative_path = os.path.relpath(local_path, local_dir) - s3_path = os.path.join( - get_minor_version(), relative_path - ) + s3_path = os.path.join(get_minor_version(), relative_path) try: - s3.head_object( - Bucket='bonobo-examples', Key=s3_path - ) + s3.head_object(Bucket="bonobo-examples", Key=s3_path) except Exception: - s3.upload_file( - local_path, - 'bonobo-examples', - s3_path, - ExtraArgs={'ACL': 'public-read'} - ) + s3.upload_file(local_path, "bonobo-examples", s3_path, ExtraArgs={"ACL": "public-read"}) diff --git a/bonobo/examples/datasets/coffeeshops.py b/bonobo/examples/datasets/coffeeshops.py index 067f593..cd52d83 100644 --- a/bonobo/examples/datasets/coffeeshops.py +++ b/bonobo/examples/datasets/coffeeshops.py @@ -1,63 +1,39 @@ -""" - -""" import sys import bonobo from bonobo import examples from bonobo.contrib.opendatasoft import OpenDataSoftAPI as ODSReader from bonobo.examples import get_services +from bonobo.structs.graphs import PartialGraph def get_graph(graph=None, *, _limit=(), _print=()): graph = graph or bonobo.Graph() - producer = graph.add_chain( - ODSReader( - dataset='liste-des-cafes-a-un-euro', - netloc='opendata.paris.fr' - ), - *_limit, - bonobo.UnpackItems(0), - bonobo.Rename( - name='nom_du_cafe', - address='adresse', - zipcode='arrondissement' - ), - bonobo.Format(city='Paris', country='France'), - bonobo.OrderFields( - [ - 'name', 'address', 'zipcode', 'city', 'country', - 'geometry', 'geoloc' - ] - ), - *_print, + producer = ( + graph.get_cursor() + >> ODSReader(dataset="liste-des-cafes-a-un-euro", netloc="opendata.paris.fr") + >> PartialGraph(*_limit) + >> bonobo.UnpackItems(0) + >> bonobo.Rename(name="nom_du_cafe", address="adresse", zipcode="arrondissement") + >> bonobo.Format(city="Paris", country="France") + >> bonobo.OrderFields(["name", "address", "zipcode", "city", "country", "geometry", "geoloc"]) + >> PartialGraph(*_print) ) # Comma separated values. - graph.add_chain( - bonobo.CsvWriter( - 'coffeeshops.csv', - fields=['name', 'address', 'zipcode', 'city'], - delimiter=',' - ), - _input=producer.output, + graph.get_cursor(producer.output) >> bonobo.CsvWriter( + "coffeeshops.csv", fields=["name", "address", "zipcode", "city"], delimiter="," ) # Standard JSON - graph.add_chain( - bonobo.JsonWriter(path='coffeeshops.json'), - _input=producer.output, - ) + graph.get_cursor(producer.output) >> bonobo.JsonWriter(path="coffeeshops.json") # Line-delimited JSON - graph.add_chain( - bonobo.LdjsonWriter(path='coffeeshops.ldjson'), - _input=producer.output, - ) + graph.get_cursor(producer.output) >> bonobo.LdjsonWriter(path="coffeeshops.ldjson") return graph -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(examples.run(get_graph, get_services)) diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index 24aaa71..5e665d5 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -25,27 +25,21 @@ from bonobo.examples import get_services try: import pycountry except ImportError as exc: - raise ImportError( - 'You must install package "pycountry" to run this example.' - ) from exc + raise ImportError('You must install package "pycountry" to run this example.') from exc -API_DATASET = 'fablabs@public-us' +API_DATASET = "fablabs@public-us" ROWS = 100 def _getlink(x): - return x.get('url', None) + return x.get("url", None) def normalize(row): result = { **row, - 'links': - list(filter(None, map(_getlink, json.loads(row.get('links'))))), - 'country': - pycountry.countries.get( - alpha_2=row.get('country_code', '').upper() - ).name, + "links": list(filter(None, map(_getlink, json.loads(row.get("links"))))), + "country": pycountry.countries.get(alpha_2=row.get("country_code", "").upper()).name, } return result @@ -58,10 +52,10 @@ def get_graph(graph=None, *, _limit=(), _print=()): normalize, bonobo.UnpackItems(0), *_print, - bonobo.JsonWriter(path='fablabs.json'), + bonobo.JsonWriter(path="fablabs.json"), ) return graph -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(examples.run(get_graph, get_services)) diff --git a/bonobo/examples/empty.py b/bonobo/examples/empty.py index 044ab43..0a14e6f 100644 --- a/bonobo/examples/empty.py +++ b/bonobo/examples/empty.py @@ -19,7 +19,7 @@ def get_graph(): return graph -if __name__ == '__main__': +if __name__ == "__main__": parser = bonobo.get_argument_parser() with bonobo.parse_args(parser): bonobo.run(get_graph()) diff --git a/bonobo/examples/environ.py b/bonobo/examples/environ.py index 280d2e1..b8cd5a5 100644 --- a/bonobo/examples/environ.py +++ b/bonobo/examples/environ.py @@ -21,7 +21,7 @@ def get_graph(): return graph -if __name__ == '__main__': +if __name__ == "__main__": parser = bonobo.get_argument_parser() with bonobo.parse_args(parser): bonobo.run(get_graph()) diff --git a/bonobo/examples/files/csv_handlers.py b/bonobo/examples/files/csv_handlers.py index 369b680..25a54cf 100644 --- a/bonobo/examples/files/csv_handlers.py +++ b/bonobo/examples/files/csv_handlers.py @@ -7,12 +7,12 @@ from bonobo.examples.files.services import get_services def get_graph(*, _limit=None, _print=False): return bonobo.Graph( - bonobo.CsvReader('coffeeshops.csv'), - *((bonobo.Limit(_limit), ) if _limit else ()), - *((bonobo.PrettyPrinter(), ) if _print else ()), - bonobo.CsvWriter('coffeeshops.csv', fs='fs.output') + bonobo.CsvReader("coffeeshops.csv"), + *((bonobo.Limit(_limit),) if _limit else ()), + *((bonobo.PrettyPrinter(),) if _print else ()), + bonobo.CsvWriter("coffeeshops.csv", fs="fs.output") ) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(examples.run(get_graph, get_services)) diff --git a/bonobo/examples/files/json_handlers.py b/bonobo/examples/files/json_handlers.py index 50908d3..f2c327e 100644 --- a/bonobo/examples/files/json_handlers.py +++ b/bonobo/examples/files/json_handlers.py @@ -9,24 +9,17 @@ def get_graph(*, _limit=None, _print=False): graph = bonobo.Graph() trunk = graph.add_chain( - bonobo.JsonReader('theaters.json', fs='fs.static'), - *((bonobo.Limit(_limit), ) if _limit else ()), + bonobo.JsonReader("theaters.json", fs="fs.static"), *((bonobo.Limit(_limit),) if _limit else ()) ) if _print: graph.add_chain(bonobo.PrettyPrinter(), _input=trunk.output) - graph.add_chain( - bonobo.JsonWriter('theaters.output.json', fs='fs.output'), - _input=trunk.output - ) - graph.add_chain( - bonobo.LdjsonWriter('theaters.output.ldjson', fs='fs.output'), - _input=trunk.output - ) + graph.add_chain(bonobo.JsonWriter("theaters.output.json", fs="fs.output"), _input=trunk.output) + graph.add_chain(bonobo.LdjsonWriter("theaters.output.ldjson", fs="fs.output"), _input=trunk.output) return graph -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(examples.run(get_graph, get_services)) diff --git a/bonobo/examples/files/pickle_handlers.py b/bonobo/examples/files/pickle_handlers.py index 4557686..81c3751 100644 --- a/bonobo/examples/files/pickle_handlers.py +++ b/bonobo/examples/files/pickle_handlers.py @@ -1,4 +1,4 @@ -''' +""" This example shows how a different file system service can be injected into a transformation (as compressing pickled objects often makes sense anyways). The pickle itself contains a list of lists as follows: @@ -25,7 +25,7 @@ https://www.kaggle.com/uciml/sms-spam-collection-dataset/downloads/sms-spam-coll The transformation (1) reads the pickled data, (2) marks and shortens messages categorized as spam, and (3) prints the output. -''' +""" import sys @@ -36,14 +36,12 @@ from bonobo import examples def cleanse_sms(category, sms): - if category == 'spam': - sms_clean = '**MARKED AS SPAM** ' + sms[0:50] + ( - '...' if len(sms) > 50 else '' - ) - elif category == 'ham': + if category == "spam": + sms_clean = "**MARKED AS SPAM** " + sms[0:50] + ("..." if len(sms) > 50 else "") + elif category == "ham": sms_clean = sms else: - raise ValueError('Unknown category {!r}.'.format(category)) + raise ValueError("Unknown category {!r}.".format(category)) return category, sms, sms_clean @@ -53,7 +51,7 @@ def get_graph(*, _limit=(), _print=()): graph.add_chain( # spam.pkl is within the gzipped tarball - bonobo.PickleReader('spam.pkl'), + bonobo.PickleReader("spam.pkl"), *_limit, cleanse_sms, *_print, @@ -63,11 +61,8 @@ def get_graph(*, _limit=(), _print=()): def get_services(): - return { - **examples.get_services(), 'fs': - TarFS(bonobo.get_examples_path('datasets', 'static', 'spam.tgz')) - } + return {**examples.get_services(), "fs": TarFS(bonobo.get_examples_path("datasets", "static", "spam.tgz"))} -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(examples.run(get_graph, get_services)) diff --git a/bonobo/examples/files/services.py b/bonobo/examples/files/services.py index fc8a42f..16d9e15 100644 --- a/bonobo/examples/files/services.py +++ b/bonobo/examples/files/services.py @@ -2,7 +2,4 @@ from bonobo import examples, open_fs def get_services(): - return { - **examples.get_services(), - 'fs.output': open_fs(), - } + return {**examples.get_services(), "fs.output": open_fs()} diff --git a/bonobo/examples/files/text_handlers.py b/bonobo/examples/files/text_handlers.py index be37c4a..5b326f5 100644 --- a/bonobo/examples/files/text_handlers.py +++ b/bonobo/examples/files/text_handlers.py @@ -7,20 +7,20 @@ from bonobo.examples.files.services import get_services def skip_comments(line): line = line.strip() - if not line.startswith('#'): + if not line.startswith("#"): yield line def get_graph(*, _limit=(), _print=()): return bonobo.Graph( - bonobo.FileReader('passwd.txt', fs='fs.static'), + bonobo.FileReader("passwd.txt", fs="fs.static"), skip_comments, *_limit, - lambda s: s.split(':')[0], + lambda s: s.split(":")[0], *_print, - bonobo.FileWriter('usernames.txt', fs='fs.output'), + bonobo.FileWriter("usernames.txt", fs="fs.output"), ) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(examples.run(get_graph, get_services)) diff --git a/bonobo/examples/types/__main__.py b/bonobo/examples/types/__main__.py index ccda1a9..e06a106 100644 --- a/bonobo/examples/types/__main__.py +++ b/bonobo/examples/types/__main__.py @@ -1,7 +1,7 @@ import bonobo from bonobo.examples.types.strings import get_graph -if __name__ == '__main__': +if __name__ == "__main__": parser = bonobo.get_argument_parser() with bonobo.parse_args(parser): bonobo.run(get_graph()) diff --git a/bonobo/examples/types/strings.py b/bonobo/examples/types/strings.py index 6675a77..3e6272e 100644 --- a/bonobo/examples/types/strings.py +++ b/bonobo/examples/types/strings.py @@ -18,13 +18,13 @@ import bonobo def extract(): - yield 'foo' - yield 'bar' - yield 'baz' + yield "foo" + yield "bar" + yield "baz" def transform(s): - return '{} ({})'.format(s.title(), randint(10, 99)) + return "{} ({})".format(s.title(), randint(10, 99)) def load(s): @@ -35,7 +35,7 @@ def get_graph(): return bonobo.Graph(extract, transform, load) -if __name__ == '__main__': +if __name__ == "__main__": parser = bonobo.get_argument_parser() with bonobo.parse_args(parser): bonobo.run(get_graph()) diff --git a/bonobo/execution/contexts/__init__.py b/bonobo/execution/contexts/__init__.py index 41e811e..18ddbe2 100644 --- a/bonobo/execution/contexts/__init__.py +++ b/bonobo/execution/contexts/__init__.py @@ -9,8 +9,4 @@ from bonobo.execution.contexts.graph import GraphExecutionContext from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.execution.contexts.plugin import PluginExecutionContext -__all__ = [ - 'GraphExecutionContext', - 'NodeExecutionContext', - 'PluginExecutionContext', -] +__all__ = ["GraphExecutionContext", "NodeExecutionContext", "PluginExecutionContext"] diff --git a/bonobo/execution/contexts/base.py b/bonobo/execution/contexts/base.py index 25cca26..c6117d1 100644 --- a/bonobo/execution/contexts/base.py +++ b/bonobo/execution/contexts/base.py @@ -62,12 +62,12 @@ class Lifecycle: """ if self._defunct: - return '!' + return "!" if not self.started: - return ' ' + return " " if not self.stopped: - return '+' - return '-' + return "+" + return "-" def __enter__(self): self.start() @@ -78,31 +78,31 @@ class Lifecycle: def get_flags_as_string(self): if self._defunct: - return term.red('[defunct]') + return term.red("[defunct]") if self.killed: - return term.lightred('[killed]') + return term.lightred("[killed]") if self.stopped: - return term.lightblack('[done]') - return '' + return term.lightblack("[done]") + return "" def start(self): if self.started: - raise RuntimeError('This context is already started ({}).'.format(get_name(self))) + raise RuntimeError("This context is already started ({}).".format(get_name(self))) self._started = True def stop(self): if not self.started: - raise RuntimeError('This context cannot be stopped as it never started ({}).'.format(get_name(self))) + raise RuntimeError("This context cannot be stopped as it never started ({}).".format(get_name(self))) self._stopped = True def kill(self): if not self.started: - raise RuntimeError('Cannot kill an unstarted context.') + raise RuntimeError("Cannot kill an unstarted context.") if self.stopped: - raise RuntimeError('Cannot kill a stopped context.') + raise RuntimeError("Cannot kill a stopped context.") self._killed = True @@ -119,10 +119,10 @@ class Lifecycle: def as_dict(self): return { - 'status': self.status, - 'name': self.name, - 'stats': self.get_statistics_as_string(), - 'flags': self.get_flags_as_string(), + "status": self.status, + "name": self.name, + "stats": self.get_statistics_as_string(), + "flags": self.get_flags_as_string(), } diff --git a/bonobo/execution/contexts/graph.py b/bonobo/execution/contexts/graph.py index e5c6554..812c29a 100644 --- a/bonobo/execution/contexts/graph.py +++ b/bonobo/execution/contexts/graph.py @@ -63,7 +63,7 @@ class BaseGraphExecutionContext(BaseContext): self.services = create_container(services) # Probably not a good idea to use it unless you really know what you're doing. But you can access the context. - self.services['__graph_context'] = self + self.services["__graph_context"] = self for i, node_context in enumerate(self): outputs = self.graph.outputs_of(i) diff --git a/bonobo/execution/contexts/node.py b/bonobo/execution/contexts/node.py index 443447d..9248356 100644 --- a/bonobo/execution/contexts/node.py +++ b/bonobo/execution/contexts/node.py @@ -19,7 +19,7 @@ from bonobo.util.statistics import WithStatistics logger = logging.getLogger(__name__) -UnboundArguments = namedtuple('UnboundArguments', ['args', 'kwargs']) +UnboundArguments = namedtuple("UnboundArguments", ["args", "kwargs"]) class NodeExecutionContext(BaseContext, WithStatistics): @@ -46,13 +46,13 @@ class NodeExecutionContext(BaseContext, WithStatistics): :param _outputs: output queues (optional) """ BaseContext.__init__(self, wrapped, parent=parent) - WithStatistics.__init__(self, 'in', 'out', 'err', 'warn') + WithStatistics.__init__(self, "in", "out", "err", "warn") # Services: how we'll access external dependencies if services: if self.parent: raise RuntimeError( - 'Having services defined both in GraphExecutionContext and child NodeExecutionContext is not supported, for now.' + "Having services defined both in GraphExecutionContext and child NodeExecutionContext is not supported, for now." ) self.services = create_container(services) else: @@ -70,11 +70,11 @@ class NodeExecutionContext(BaseContext, WithStatistics): self._stack = None def __str__(self): - return self.__name__ + self.get_statistics_as_string(prefix=' ') + return self.__name__ + self.get_statistics_as_string(prefix=" ") def __repr__(self): name, type_name = get_name(self), get_name(type(self)) - return '<{}({}{}){}>'.format(type_name, self.status, name, self.get_statistics_as_string(prefix=' ')) + return "<{}({}{}){}>".format(type_name, self.status, name, self.get_statistics_as_string(prefix=" ")) def start(self): """ @@ -97,13 +97,13 @@ class NodeExecutionContext(BaseContext, WithStatistics): # Not normal to have a partially configured object here, so let's warn the user instead of having get into # the hard trouble of understanding that by himself. raise TypeError( - 'Configurables should be instanciated before execution starts.\nGot {!r}.\n'.format( + "Configurables should be instanciated before execution starts.\nGot {!r}.\n".format( self.wrapped ) ) from exc else: raise TypeError( - 'Configurables should be instanciated before execution starts.\nGot {!r}.\n'.format( + "Configurables should be instanciated before execution starts.\nGot {!r}.\n".format( self.wrapped ) ) @@ -120,7 +120,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): The actual infinite loop for this transformation. """ - logger.debug('Node loop starts for {!r}.'.format(self)) + logger.debug("Node loop starts for {!r}.".format(self)) while self.should_loop: try: @@ -128,7 +128,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): except InactiveReadableError: break - logger.debug('Node loop ends for {!r}.'.format(self)) + logger.debug("Node loop ends for {!r}.".format(self)) def step(self): try: @@ -137,10 +137,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): raise except Empty: sleep(TICK_PERIOD) # XXX: How do we determine this constant? - except ( - NotImplementedError, - UnrecoverableError, - ): + except (NotImplementedError, UnrecoverableError): self.fatal(sys.exc_info()) # exit loop except Exception: # pylint: disable=broad-except self.error(sys.exc_info()) # does not exit loop @@ -208,20 +205,20 @@ class NodeExecutionContext(BaseContext, WithStatistics): def set_input_type(self, input_type): if self._input_type is not None: - raise RuntimeError('Cannot override input type, already have %r.', self._input_type) + raise RuntimeError("Cannot override input type, already have %r.", self._input_type) if type(input_type) is not type: - raise UnrecoverableTypeError('Input types must be regular python types.') + raise UnrecoverableTypeError("Input types must be regular python types.") if not issubclass(input_type, tuple): - raise UnrecoverableTypeError('Input types must be subclasses of tuple (and act as tuples).') + raise UnrecoverableTypeError("Input types must be subclasses of tuple (and act as tuples).") self._input_type = input_type def get_input_fields(self): - return self._input_type._fields if self._input_type and hasattr(self._input_type, '_fields') else None + return self._input_type._fields if self._input_type and hasattr(self._input_type, "_fields") else None - def set_input_fields(self, fields, typename='Bag'): + def set_input_fields(self, fields, typename="Bag"): self.set_input_type(BagType(typename, fields)) ### Output type and fields @@ -231,20 +228,20 @@ class NodeExecutionContext(BaseContext, WithStatistics): def set_output_type(self, output_type): if self._output_type is not None: - raise RuntimeError('Cannot override output type, already have %r.', self._output_type) + raise RuntimeError("Cannot override output type, already have %r.", self._output_type) if type(output_type) is not type: - raise UnrecoverableTypeError('Output types must be regular python types.') + raise UnrecoverableTypeError("Output types must be regular python types.") if not issubclass(output_type, tuple): - raise UnrecoverableTypeError('Output types must be subclasses of tuple (and act as tuples).') + raise UnrecoverableTypeError("Output types must be subclasses of tuple (and act as tuples).") self._output_type = output_type def get_output_fields(self): - return self._output_type._fields if self._output_type and hasattr(self._output_type, '_fields') else None + return self._output_type._fields if self._output_type and hasattr(self._output_type, "_fields") else None - def set_output_fields(self, fields, typename='Bag'): + def set_output_fields(self, fields, typename="Bag"): self.set_output_type(BagType(typename, fields)) ### Attributes @@ -273,11 +270,11 @@ class NodeExecutionContext(BaseContext, WithStatistics): self.step() def error(self, exc_info, *, level=logging.ERROR): - self.increment('err') + self.increment("err") super().error(exc_info, level=level) def fatal(self, exc_info, *, level=logging.CRITICAL): - self.increment('err') + self.increment("err") super().fatal(exc_info, level=level) self.input.shutdown() @@ -306,8 +303,9 @@ class NodeExecutionContext(BaseContext, WithStatistics): input_bag = self._input_type(*input_bag) except Exception as exc: raise UnrecoverableTypeError( - 'Input type changed to incompatible type between calls to {!r}.\nGot {!r} which is not of type {!r}.'. - format(self.wrapped, input_bag, self._input_type) + "Input type changed to incompatible type between calls to {!r}.\nGot {!r} which is not of type {!r}.".format( + self.wrapped, input_bag, self._input_type + ) ) from exc # Store or check input length, which is a soft fallback in case we're just using tuples @@ -315,12 +313,12 @@ class NodeExecutionContext(BaseContext, WithStatistics): self._input_length = len(input_bag) elif len(input_bag) != self._input_length: raise UnrecoverableTypeError( - 'Input length changed between calls to {!r}.\nExpected {} but got {}: {!r}.'.format( + "Input length changed between calls to {!r}.\nExpected {} but got {}: {!r}.".format( self.wrapped, self._input_length, len(input_bag), input_bag ) ) - self.increment('in') # XXX should that go before type check ? + self.increment("in") # XXX should that go before type check ? return input_bag @@ -366,7 +364,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): """ if not _control: - self.increment('out') + self.increment("out") for output in self.outputs: output.put(value) @@ -406,8 +404,9 @@ class AsyncNodeExecutionContext(NodeExecutionContext): input_bag = self._input_type(*input_bag) except Exception as exc: raise UnrecoverableTypeError( - 'Input type changed to incompatible type between calls to {!r}.\nGot {!r} which is not of type {!r}.'. - format(self.wrapped, input_bag, self._input_type) + "Input type changed to incompatible type between calls to {!r}.\nGot {!r} which is not of type {!r}.".format( + self.wrapped, input_bag, self._input_type + ) ) from exc # Store or check input length, which is a soft fallback in case we're just using tuples @@ -415,12 +414,12 @@ class AsyncNodeExecutionContext(NodeExecutionContext): self._input_length = len(input_bag) elif len(input_bag) != self._input_length: raise UnrecoverableTypeError( - 'Input length changed between calls to {!r}.\nExpected {} but got {}: {!r}.'.format( + "Input length changed between calls to {!r}.\nExpected {} but got {}: {!r}.".format( self.wrapped, self._input_length, len(input_bag), input_bag ) ) - self.increment('in') # XXX should that go before type check ? + self.increment("in") # XXX should that go before type check ? return input_bag @@ -443,18 +442,18 @@ def split_token(output): flags, i, len_output, data_allowed = set(), 0, len(output), True while i < len_output and isflag(output[i]): if output[i].must_be_first and i: - raise ValueError('{} flag must be first.'.format(output[i])) + raise ValueError("{} flag must be first.".format(output[i])) if i and output[i - 1].must_be_last: - raise ValueError('{} flag must be last.'.format(output[i - 1])) + raise ValueError("{} flag must be last.".format(output[i - 1])) if output[i] in flags: - raise ValueError('Duplicate flag {}.'.format(output[i])) + raise ValueError("Duplicate flag {}.".format(output[i])) flags.add(output[i]) data_allowed &= output[i].allows_data i += 1 output = output[i:] if not data_allowed and len(output): - raise ValueError('Output data provided after a flag that does not allow data.') + raise ValueError("Output data provided after a flag that does not allow data.") return flags, output @@ -465,7 +464,7 @@ def concat_types(t1, l1, t2, l2): if t1 == t2 == tuple: return tuple - f1 = t1._fields if hasattr(t1, '_fields') else tuple(range(l1)) - f2 = t2._fields if hasattr(t2, '_fields') else tuple(range(l2)) + f1 = t1._fields if hasattr(t1, "_fields") else tuple(range(l1)) + f2 = t2._fields if hasattr(t2, "_fields") else tuple(range(l2)) - return BagType('Inherited', f1 + f2) + return BagType("Inherited", f1 + f2) diff --git a/bonobo/execution/events.py b/bonobo/execution/events.py index 3269bfc..4772515 100644 --- a/bonobo/execution/events.py +++ b/bonobo/execution/events.py @@ -27,12 +27,12 @@ from whistle import Event -START = 'execution.start' -STARTED = 'execution.started' -TICK = 'execution.tick' -STOP = 'execution.stop' -STOPPED = 'execution.stopped' -KILL = 'execution.kill' +START = "execution.start" +STARTED = "execution.started" +TICK = "execution.tick" +STOP = "execution.stop" +STOPPED = "execution.stopped" +KILL = "execution.kill" class ExecutionEvent(Event): diff --git a/bonobo/execution/strategies/__init__.py b/bonobo/execution/strategies/__init__.py index 318550e..cb9944a 100644 --- a/bonobo/execution/strategies/__init__.py +++ b/bonobo/execution/strategies/__init__.py @@ -6,22 +6,23 @@ In the future, the two strategies that would really benefit bonobo are subproces at home if you want to give it a shot. """ -from bonobo.execution.strategies.executor import ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy, \ - AsyncThreadPoolExecutorStrategy +from bonobo.execution.strategies.executor import ( + ProcessPoolExecutorStrategy, + ThreadPoolExecutorStrategy, + AsyncThreadPoolExecutorStrategy, +) from bonobo.execution.strategies.naive import NaiveStrategy -__all__ = [ - 'create_strategy', -] +__all__ = ["create_strategy"] STRATEGIES = { - 'naive': NaiveStrategy, - 'processpool': ProcessPoolExecutorStrategy, - 'threadpool': ThreadPoolExecutorStrategy, - 'aio_threadpool': AsyncThreadPoolExecutorStrategy, + "naive": NaiveStrategy, + "processpool": ProcessPoolExecutorStrategy, + "threadpool": ThreadPoolExecutorStrategy, + "aio_threadpool": AsyncThreadPoolExecutorStrategy, } -DEFAULT_STRATEGY = 'threadpool' +DEFAULT_STRATEGY = "threadpool" def create_strategy(name=None): @@ -40,13 +41,13 @@ def create_strategy(name=None): if name is None: name = DEFAULT_STRATEGY - logging.debug('Creating execution strategy {!r}...'.format(name)) + logging.debug("Creating execution strategy {!r}...".format(name)) try: factory = STRATEGIES[name] except KeyError as exc: raise RuntimeError( - 'Invalid strategy {}. Available choices: {}.'.format(repr(name), ', '.join(sorted(STRATEGIES.keys()))) + "Invalid strategy {}. Available choices: {}.".format(repr(name), ", ".join(sorted(STRATEGIES.keys()))) ) from exc return factory() diff --git a/bonobo/execution/strategies/base.py b/bonobo/execution/strategies/base.py index 69e1d65..cfba064 100644 --- a/bonobo/execution/strategies/base.py +++ b/bonobo/execution/strategies/base.py @@ -6,6 +6,7 @@ class Strategy: Base class for execution strategies. """ + GraphExecutionContextType = GraphExecutionContext def __init__(self, GraphExecutionContextType=None): @@ -13,7 +14,7 @@ class Strategy: def create_graph_execution_context(self, graph, *args, GraphExecutionContextType=None, **kwargs): if not len(graph): - raise ValueError('You provided an empty graph, which does not really make sense. Please add some nodes.') + raise ValueError("You provided an empty graph, which does not really make sense. Please add some nodes.") return (GraphExecutionContextType or self.GraphExecutionContextType)(graph, *args, **kwargs) def execute(self, graph, *args, **kwargs): diff --git a/bonobo/execution/strategies/executor.py b/bonobo/execution/strategies/executor.py index ae9b259..17c3ab3 100644 --- a/bonobo/execution/strategies/executor.py +++ b/bonobo/execution/strategies/executor.py @@ -36,14 +36,14 @@ class ExecutorStrategy(Strategy): try: context.start(self.get_starter(executor, futures)) except Exception: - logger.critical('Exception caught while starting execution context.', exc_info=sys.exc_info()) + logger.critical("Exception caught while starting execution context.", exc_info=sys.exc_info()) while context.alive: try: context.tick() except KeyboardInterrupt: logging.getLogger(__name__).warning( - 'KeyboardInterrupt received. Trying to terminate the nodes gracefully.' + "KeyboardInterrupt received. Trying to terminate the nodes gracefully." ) context.kill() break @@ -61,13 +61,13 @@ class ExecutorStrategy(Strategy): node.loop() except Exception: logging.getLogger(__name__).critical( - 'Critical error in threadpool node starter.', exc_info=sys.exc_info() + "Critical error in threadpool node starter.", exc_info=sys.exc_info() ) try: futures.append(executor.submit(_runner)) except Exception: - logging.getLogger(__name__).critical('futures.append', exc_info=sys.exc_info()) + logging.getLogger(__name__).critical("futures.append", exc_info=sys.exc_info()) return starter @@ -85,7 +85,7 @@ class AsyncThreadPoolExecutorStrategy(ThreadPoolExecutorStrategy): def __init__(self, GraphExecutionContextType=None): if not settings.ALPHA.get(): raise NotImplementedError( - '{} is experimental, you need to explicitely activate it using ALPHA=True in system env.'.format( + "{} is experimental, you need to explicitely activate it using ALPHA=True in system env.".format( get_name(self) ) ) diff --git a/bonobo/nodes/__init__.py b/bonobo/nodes/__init__.py index 2cdd1e9..843a3bb 100644 --- a/bonobo/nodes/__init__.py +++ b/bonobo/nodes/__init__.py @@ -5,4 +5,4 @@ from bonobo.nodes.io import * from bonobo.nodes.io import __all__ as _all_io from bonobo.nodes.throttle import RateLimited -__all__ = _all_basics + _all_io + ['Filter', 'RateLimited'] +__all__ = _all_basics + _all_io + ["Filter", "RateLimited"] diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 7bdfe24..9104a7c 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -13,18 +13,18 @@ from bonobo.util.term import CLEAR_EOL from mondrian import term __all__ = [ - 'FixedWindow', - 'Format', - 'Limit', - 'OrderFields', - 'PrettyPrinter', - 'Rename', - 'SetFields', - 'Tee', - 'UnpackItems', - 'count', - 'identity', - 'noop', + "FixedWindow", + "Format", + "Limit", + "OrderFields", + "PrettyPrinter", + "Rename", + "SetFields", + "Tee", + "UnpackItems", + "count", + "identity", + "noop", ] @@ -43,6 +43,7 @@ class Limit(Configurable): TODO: simplify into a closure building factory? """ + limit = Option(positional=True, default=10) @ContextProcessor @@ -69,7 +70,7 @@ def Tee(f): def _shorten(s, w): if w and len(s) > w: - s = s[0:w - 3] + '...' + s = s[0 : w - 3] + "..." return s @@ -78,28 +79,31 @@ class PrettyPrinter(Configurable): int, default=term.get_size()[0], required=False, - __doc__=''' + __doc__=""" If set, truncates the output values longer than this to this width. - ''' + """, ) filter = Method( - default= - (lambda self, index, key, value: (value is not None) and (not isinstance(key, str) or not key.startswith('_'))), - __doc__=''' + default=( + lambda self, index, key, value: (value is not None) + and (not isinstance(key, str) or not key.startswith("_")) + ), + __doc__=""" A filter that determine what to print. Default is to ignore any key starting with an underscore and none values. - ''' + """, ) @ContextProcessor def context(self, context): - context.setdefault('_jupyter_html', None) + context.setdefault("_jupyter_html", None) yield context if context._jupyter_html is not None: from IPython.display import display, HTML - display(HTML('\n'.join([''] + context._jupyter_html + ['
']))) + + display(HTML("\n".join([""] + context._jupyter_html + ["
"]))) def __call__(self, context, *args, **kwargs): if not settings.QUIET: @@ -120,49 +124,44 @@ class PrettyPrinter(Configurable): def format_quiet(self, index, key, value, *, fields=None): # XXX should we implement argnames here ? - return ' '.join(((' ' if index else '-'), str(key), ':', str(value).strip())) + return " ".join(((" " if index else "-"), str(key), ":", str(value).strip())) def print_console(self, context, *args, **kwargs): - print('\u250c') + print("\u250c") for index, (key, value) in enumerate(itertools.chain(enumerate(args), kwargs.items())): if self.filter(index, key, value): print(self.format_console(index, key, value, fields=context.get_input_fields())) - print('\u2514') + print("\u2514") def format_console(self, index, key, value, *, fields=None): fields = fields or [] if not isinstance(key, str): if len(fields) > key and str(key) != str(fields[key]): - key = '{}{}'.format(fields[key], term.lightblack('[{}]'.format(key))) + key = "{}{}".format(fields[key], term.lightblack("[{}]".format(key))) else: key = str(index) - prefix = '\u2502 {} = '.format(key) + prefix = "\u2502 {} = ".format(key) prefix_length = len(prefix) def indent(text, prefix): for i, line in enumerate(text.splitlines()): - yield (prefix if i else '') + line + CLEAR_EOL + '\n' + yield (prefix if i else "") + line + CLEAR_EOL + "\n" - repr_of_value = ''.join( - indent(pprint.pformat(value, width=self.max_width - prefix_length), '\u2502' + ' ' * (len(prefix) - 1)) + repr_of_value = "".join( + indent(pprint.pformat(value, width=self.max_width - prefix_length), "\u2502" + " " * (len(prefix) - 1)) ).strip() - return '{}{}{}'.format(prefix, repr_of_value.replace('\n', CLEAR_EOL + '\n'), CLEAR_EOL) + return "{}{}{}".format(prefix, repr_of_value.replace("\n", CLEAR_EOL + "\n"), CLEAR_EOL) def print_jupyter(self, context, *args): if not context._jupyter_html: context._jupyter_html = [ - '', - *map('{}'.format, map(html.escape, map(str, - context.get_input_fields() or range(len(args))))), - '', + "", + *map("{}".format, map(html.escape, map(str, context.get_input_fields() or range(len(args))))), + "", ] - context._jupyter_html += [ - '', - *map('{}'.format, map(html.escape, map(repr, args))), - '', - ] + context._jupyter_html += ["", *map("{}".format, map(html.escape, map(repr, args))), ""] @use_no_input @@ -211,7 +210,7 @@ def OrderFields(fields): @use_raw_input def _OrderFields(context, row): nonlocal fields - context.setdefault('remaining', None) + context.setdefault("remaining", None) if not context.output_type: context.remaining = list(sorted(set(context.get_input_fields()) - set(fields))) context.set_output_fields(fields + context.remaining) diff --git a/bonobo/nodes/io/__init__.py b/bonobo/nodes/io/__init__.py index 1369ed2..c375c91 100644 --- a/bonobo/nodes/io/__init__.py +++ b/bonobo/nodes/io/__init__.py @@ -6,14 +6,14 @@ from .json import JsonReader, JsonWriter, LdjsonReader, LdjsonWriter from .pickle import PickleReader, PickleWriter __all__ = [ - 'CsvReader', - 'CsvWriter', - 'FileReader', - 'FileWriter', - 'JsonReader', - 'JsonWriter', - 'LdjsonReader', - 'LdjsonWriter', - 'PickleReader', - 'PickleWriter', + "CsvReader", + "CsvWriter", + "FileReader", + "FileWriter", + "JsonReader", + "JsonWriter", + "LdjsonReader", + "LdjsonWriter", + "PickleReader", + "PickleWriter", ] diff --git a/bonobo/nodes/io/base.py b/bonobo/nodes/io/base.py index d2cca60..26d84ef 100644 --- a/bonobo/nodes/io/base.py +++ b/bonobo/nodes/io/base.py @@ -13,22 +13,39 @@ class FileHandler(Configurable): """ path = Option( - str, required=True, positional=True, __doc__=''' + str, + required=True, + positional=True, + __doc__=""" Path to use within the provided filesystem. - ''' + """, ) # type: str - eol = Option(str, default='\n', __doc__=''' + eol = Option( + str, + default="\n", + __doc__=""" Character to use as line separator. - ''') # type: str - mode = Option(str, __doc__=''' + """, + ) # type: str + mode = Option( + str, + __doc__=""" What mode to use for open() call. - ''') # type: str - encoding = Option(str, default='utf-8', __doc__=''' + """, + ) # type: str + encoding = Option( + str, + default="utf-8", + __doc__=""" Encoding. - ''') # type: str - fs = Service('fs', __doc__=''' + """, + ) # type: str + fs = Service( + "fs", + __doc__=""" The filesystem instance to use. - ''') # type: str + """, + ) # type: str @ContextProcessor def file(self, context, *, fs): diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index 7900dca..c97407f 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -35,18 +35,18 @@ class CsvHandler(FileHandler): quoting = Option(int, default=csv.excel.quoting, required=False) # Fields (renamed from headers) - headers = RenamedOption('fields') + headers = RenamedOption("fields") fields = Option(ensure_tuple, required=False) def get_dialect_kwargs(self): return { - 'delimiter': self.delimiter, - 'quotechar': self.quotechar, - 'escapechar': self.escapechar, - 'doublequote': self.doublequote, - 'skipinitialspace': self.skipinitialspace, - 'lineterminator': self.lineterminator, - 'quoting': self.quoting, + "delimiter": self.delimiter, + "quotechar": self.quotechar, + "escapechar": self.escapechar, + "doublequote": self.doublequote, + "skipinitialspace": self.skipinitialspace, + "lineterminator": self.lineterminator, + "quoting": self.quoting, } @@ -59,25 +59,25 @@ class CsvReader(FileReader, CsvHandler): skip = Option( int, default=0, - __doc__=''' + __doc__=""" If set and greater than zero, the reader will skip this amount of lines. - ''' + """, ) @Method( positional=False, - __doc__=''' + __doc__=""" Builds the CSV reader, a.k.a an object we can iterate, each iteration giving one line of fields, as an iterable. Defaults to builtin csv.reader(...), but can be overriden to fit your special needs. - ''' + """, ) def reader_factory(self, file): return csv.reader(file, **self.get_dialect_kwargs()) def read(self, file, context, *, fs): - context.setdefault('skipped', 0) + context.setdefault("skipped", 0) reader = self.reader_factory(file) skip = self.skip @@ -96,18 +96,18 @@ class CsvReader(FileReader, CsvHandler): @use_context class CsvWriter(FileWriter, CsvHandler): @Method( - __doc__=''' + __doc__=""" Builds the CSV writer, a.k.a an object we can pass a field collection to be written as one line in the target file. Defaults to builtin csv.writer(...).writerow, but can be overriden to fit your special needs. - ''' + """ ) def writer_factory(self, file): return csv.writer(file, **self.get_dialect_kwargs()).writerow def write(self, file, context, *values, fs): - context.setdefault('lineno', 0) + context.setdefault("lineno", 0) fields = context.get_input_fields() if not context.lineno: @@ -120,7 +120,7 @@ class CsvWriter(FileWriter, CsvHandler): if fields: if len(values) != len(fields): raise ValueError( - 'Values length differs from input fields length. Expected: {}. Got: {}. Values: {!r}.'.format( + "Values length differs from input fields length. Expected: {}. Got: {}. Values: {!r}.".format( len(fields), len(values), values ) ) diff --git a/bonobo/nodes/io/file.py b/bonobo/nodes/io/file.py index fe5ca37..edc2993 100644 --- a/bonobo/nodes/io/file.py +++ b/bonobo/nodes/io/file.py @@ -12,24 +12,28 @@ class FileReader(Reader, FileHandler): present. Extending it is usually the right way to create more specific file readers (like json, csv, etc.) """ - mode = Option(str, default='r', __doc__=''' + mode = Option( + str, + default="r", + __doc__=""" What mode to use for open() call. - ''') # type: str + """, + ) # type: str output_fields = Option( ensure_tuple, required=False, - __doc__=''' + __doc__=""" Specify the field names of output lines. Mutually exclusive with "output_type". - ''' + """, ) output_type = Option( required=False, - __doc__=''' + __doc__=""" Specify the type of output lines. Mutually exclusive with "output_fields". - ''' + """, ) @ContextProcessor @@ -43,7 +47,7 @@ class FileReader(Reader, FileHandler): output_type = self.output_type if output_fields and output_type: - raise UnrecoverableError('Cannot specify both output_fields and output_type option.') + raise UnrecoverableError("Cannot specify both output_fields and output_type option.") if self.output_type: context.set_output_type(self.output_type) @@ -72,16 +76,20 @@ class FileWriter(Writer, FileHandler): usually the right way to create more specific file writers (like json, csv, etc.) """ - mode = Option(str, default='w+', __doc__=''' + mode = Option( + str, + default="w+", + __doc__=""" What mode to use for open() call. - ''') # type: str + """, + ) # type: str def write(self, file, context, line, *, fs): """ Write a row on the next line of opened file in context. """ - context.setdefault('lineno', 0) - self._write_line(file, (self.eol if context.lineno else '') + line) + context.setdefault("lineno", 0) + self._write_line(file, (self.eol if context.lineno else "") + line) context.lineno += 1 return NOT_MODIFIED diff --git a/bonobo/nodes/io/json.py b/bonobo/nodes/io/json.py index 86d3262..081734c 100644 --- a/bonobo/nodes/io/json.py +++ b/bonobo/nodes/io/json.py @@ -9,13 +9,13 @@ from bonobo.nodes.io.file import FileReader, FileWriter class JsonHandler(FileHandler): - eol = ',\n' - prefix, suffix = '[', ']' + eol = ",\n" + prefix, suffix = "[", "]" class LdjsonHandler(FileHandler): - eol = '\n' - prefix, suffix = '', '' + eol = "\n" + prefix, suffix = "", "" class JsonReader(JsonHandler, FileReader): @@ -58,16 +58,16 @@ class JsonWriter(JsonHandler, FileWriter): :param ctx: :param row: """ - context.setdefault('lineno', 0) + context.setdefault("lineno", 0) fields = context.get_input_fields() if fields: - prefix = self.eol if context.lineno else '' + prefix = self.eol if context.lineno else "" self._write_line(file, prefix + json.dumps(OrderedDict(zip(fields, args)))) context.lineno += 1 else: for arg in args: - prefix = self.eol if context.lineno else '' + prefix = self.eol if context.lineno else "" self._write_line(file, prefix + json.dumps(arg)) context.lineno += 1 diff --git a/bonobo/nodes/io/pickle.py b/bonobo/nodes/io/pickle.py index da96a6d..112788d 100644 --- a/bonobo/nodes/io/pickle.py +++ b/bonobo/nodes/io/pickle.py @@ -24,7 +24,7 @@ class PickleReader(FileReader, PickleHandler): Reads a Python pickle object and yields the items in dicts. """ - mode = Option(str, default='rb') + mode = Option(str, default="rb") def read(self, file, context, *, fs): data = pickle.load(file) @@ -47,7 +47,7 @@ class PickleReader(FileReader, PickleHandler): for row in iterator: if len(row) != fields_length: - raise ValueError('Received an object with {} items, expected {}.'.format(len(row), fields_length)) + raise ValueError("Received an object with {} items, expected {}.".format(len(row), fields_length)) yield tuple(row.values() if is_dict else row) @@ -56,13 +56,13 @@ class PickleReader(FileReader, PickleHandler): @use_context class PickleWriter(FileWriter, PickleHandler): - mode = Option(str, default='wb') + mode = Option(str, default="wb") def write(self, file, context, item, *, fs): """ Write a pickled item to the opened file. """ - context.setdefault('lineno', 0) + context.setdefault("lineno", 0) file.write(pickle.dumps(item)) context.lineno += 1 return NOT_MODIFIED diff --git a/bonobo/plugins/console.py b/bonobo/plugins/console.py index 81320b5..7f619a5 100644 --- a/bonobo/plugins/console.py +++ b/bonobo/plugins/console.py @@ -35,7 +35,7 @@ class ConsoleOutputPlugin(Plugin): isatty = False # Whether we're on windows, or a real operating system. - iswindows = (sys.platform == 'win32') + iswindows = sys.platform == "win32" def __init__(self): self.isatty = self._stdout.isatty() @@ -55,9 +55,9 @@ class ConsoleOutputPlugin(Plugin): # Two options: # - move state to context # - forbid registering more than once - self.prefix = '' + self.prefix = "" self.counter = 0 - self._append_cache = '' + self._append_cache = "" self.stdout = IOBuffer() self.redirect_stdout = redirect_stdout(self._stdout if self.iswindows else self.stdout) @@ -78,13 +78,13 @@ class ConsoleOutputPlugin(Plugin): self.redirect_stderr.__exit__(None, None, None) self.redirect_stdout.__exit__(None, None, None) - def write(self, context, prefix='', rewind=True, append=None): + def write(self, context, prefix="", rewind=True, append=None): t_cnt = len(context) if not self.iswindows: - for line in self.stdout.switch().split('\n')[:-1]: + for line in self.stdout.switch().split("\n")[:-1]: print(line + CLEAR_EOL, file=self._stdout) - for line in self.stderr.switch().split('\n')[:-1]: + for line in self.stderr.switch().split("\n")[:-1]: print(line + CLEAR_EOL, file=self._stderr) alive_color = Style.BRIGHT @@ -92,31 +92,36 @@ class ConsoleOutputPlugin(Plugin): for i in context.graph.topologically_sorted_indexes: node = context[i] - name_suffix = '({})'.format(i) if settings.DEBUG.get() else '' + name_suffix = "({})".format(i) if settings.DEBUG.get() else "" liveliness_color = alive_color if node.alive else dead_color - liveliness_prefix = ' {}{}{} '.format(liveliness_color, node.status, Style.RESET_ALL) - _line = ''.join(( - liveliness_prefix, - node.name, - name_suffix, - ' ', - node.get_statistics_as_string(), - ' ', - node.get_flags_as_string(), - Style.RESET_ALL, - ' ', - )) + liveliness_prefix = " {}{}{} ".format(liveliness_color, node.status, Style.RESET_ALL) + _line = "".join( + ( + liveliness_prefix, + node.name, + name_suffix, + " ", + node.get_statistics_as_string(), + " ", + node.get_flags_as_string(), + Style.RESET_ALL, + " ", + ) + ) print(prefix + _line + CLEAR_EOL, file=self._stderr) if append: # todo handle multiline print( - ''.join(( - ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), - CLEAR_EOL - )), - file=self._stderr + "".join( + ( + " `-> ", + " ".join("{}{}{}: {}".format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), + CLEAR_EOL, + ) + ), + file=self._stderr, ) t_cnt += 1 @@ -129,16 +134,17 @@ class ConsoleOutputPlugin(Plugin): if self.counter % 10 and self._append_cache: append = self._append_cache else: - self._append_cache = append = (('Memory', '{0:.2f} Mb'.format(memory_usage())), - # ('Total time', '{0} s'.format(execution_time(harness))), - ) + self._append_cache = append = ( + ("Memory", "{0:.2f} Mb".format(memory_usage())), + # ('Total time', '{0} s'.format(execution_time(harness))), + ) else: append = () self.write(context, prefix=self.prefix, append=append, rewind=rewind) self.counter += 1 -class IOBuffer(): +class IOBuffer: """ The role of IOBuffer is to overcome the problem of multiple threads wanting to write to stdout at the same time. It works a bit like a videogame: there are two buffers, one that is used to write, and one which is used to read from. @@ -165,5 +171,6 @@ class IOBuffer(): def memory_usage(): import os, psutil + process = psutil.Process(os.getpid()) - return process.memory_info()[0] / float(2**20) + return process.memory_info()[0] / float(2 ** 20) diff --git a/bonobo/plugins/jupyter.py b/bonobo/plugins/jupyter.py index 245ac95..455a97f 100644 --- a/bonobo/plugins/jupyter.py +++ b/bonobo/plugins/jupyter.py @@ -8,9 +8,9 @@ try: import IPython.core.display except ImportError as e: logging.exception( - 'You must install Jupyter to use the bonobo Jupyter extension. Easiest way is to install the ' + "You must install Jupyter to use the bonobo Jupyter extension. Easiest way is to install the " 'optional "jupyter" dependencies with «pip install bonobo[jupyter]», but you can also install a ' - 'specific version by yourself.' + "specific version by yourself." ) diff --git a/bonobo/registry.py b/bonobo/registry.py index 8ef7773..2a5d130 100644 --- a/bonobo/registry.py +++ b/bonobo/registry.py @@ -5,23 +5,23 @@ from bonobo.nodes import ( CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter, PickleReader, PickleWriter ) -FILETYPE_CSV = 'text/csv' -FILETYPE_JSON = 'application/json' -FILETYPE_PICKLE = 'pickle' -FILETYPE_PLAIN = 'text/plain' +FILETYPE_CSV = "text/csv" +FILETYPE_JSON = "application/json" +FILETYPE_PICKLE = "pickle" +FILETYPE_PLAIN = "text/plain" -READER = 'reader' -WRITER = 'writer' +READER = "reader" +WRITER = "writer" class Registry: ALIASES = { - 'csv': FILETYPE_CSV, - 'json': FILETYPE_JSON, - 'pickle': FILETYPE_PICKLE, - 'plain': FILETYPE_PLAIN, - 'text': FILETYPE_PLAIN, - 'txt': FILETYPE_PLAIN, + "csv": FILETYPE_CSV, + "json": FILETYPE_JSON, + "pickle": FILETYPE_PICKLE, + "plain": FILETYPE_PLAIN, + "text": FILETYPE_PLAIN, + "txt": FILETYPE_PLAIN, } FACTORIES = { @@ -41,10 +41,10 @@ class Registry: def get_factory_for(self, kind, name, *, format=None): if not kind in self.FACTORIES: - raise KeyError('Unknown factory kind {!r}.'.format(kind)) + raise KeyError("Unknown factory kind {!r}.".format(kind)) if format is None and name is None: - raise RuntimeError('Cannot guess factory without at least a filename or a format.') + raise RuntimeError("Cannot guess factory without at least a filename or a format.") # Guess mimetype if possible if format is None: @@ -62,7 +62,7 @@ class Registry: if format is None or not format in self.FACTORIES[kind]: raise RuntimeError( - 'Could not resolve {kind} factory for {name} ({format}).'.format(kind=kind, name=name, format=format) + "Could not resolve {kind} factory for {name} ({format}).".format(kind=kind, name=name, format=format) ) return self.FACTORIES[kind][format] diff --git a/bonobo/settings.py b/bonobo/settings.py index 833dace..16f10a9 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -10,7 +10,7 @@ def to_bool(s): if type(s) is bool: return s if len(s): - if s.lower() in ('f', 'false', 'n', 'no', '0'): + if s.lower() in ("f", "false", "n", "no", "0"): return False return True return False @@ -40,7 +40,7 @@ class Setting: self.formatter = formatter def __repr__(self): - return ''.format(self.name, self.get()) + return "".format(self.name, self.get()) def __eq__(self, other): return self.get() == other @@ -51,7 +51,7 @@ class Setting: def set(self, value): value = self.formatter(value) if self.formatter else value if self.validator and not self.validator(value): - raise ValidationError(self, 'Invalid value {!r} for setting {!r}.'.format(value, self.name)) + raise ValidationError(self, "Invalid value {!r} for setting {!r}.".format(value, self.name)) self.value = value def set_if_true(self, value): @@ -78,40 +78,37 @@ class Setting: # Debug/verbose mode. -DEBUG = Setting('DEBUG', formatter=to_bool, default=False) +DEBUG = Setting("DEBUG", formatter=to_bool, default=False) # Profile mode. -PROFILE = Setting('PROFILE', formatter=to_bool, default=False) +PROFILE = Setting("PROFILE", formatter=to_bool, default=False) # Alpha mode. -ALPHA = Setting('ALPHA', formatter=to_bool, default=False) +ALPHA = Setting("ALPHA", formatter=to_bool, default=False) # Quiet mode. -QUIET = Setting('QUIET', formatter=to_bool, default=False) +QUIET = Setting("QUIET", formatter=to_bool, default=False) # Logging level. LOGGING_LEVEL = Setting( - 'LOGGING_LEVEL', + "LOGGING_LEVEL", formatter=logging._checkLevel, validator=logging._checkLevel, - default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO + default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO, ) # Input/Output format for transformations -IOFORMAT_ARG0 = 'arg0' -IOFORMAT_KWARGS = 'kwargs' +IOFORMAT_ARG0 = "arg0" +IOFORMAT_KWARGS = "kwargs" -IOFORMATS = { - IOFORMAT_ARG0, - IOFORMAT_KWARGS, -} +IOFORMATS = {IOFORMAT_ARG0, IOFORMAT_KWARGS} -IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__) +IOFORMAT = Setting("IOFORMAT", default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__) def check(): if DEBUG.get() and QUIET.get(): - raise RuntimeError('I cannot be verbose and quiet at the same time.') + raise RuntimeError("I cannot be verbose and quiet at the same time.") clear_all = Setting.clear_all diff --git a/bonobo/structs/graphs.py b/bonobo/structs/graphs.py index 62ccacb..430127b 100644 --- a/bonobo/structs/graphs.py +++ b/bonobo/structs/graphs.py @@ -9,24 +9,51 @@ from graphviz.dot import Digraph from bonobo.constants import BEGIN from bonobo.util import get_name -GraphRange = namedtuple('GraphRange', ['graph', 'input', 'output']) +GraphRange = namedtuple("GraphRange", ["graph", "input", "output"]) + class GraphCursor: - def __init__(self, graph, node): + @property + def input(self): + return self.first + + @property + def output(self): + return self.last + + def __init__(self, graph, *, first=None, last=None): self.graph = graph - self.node = node + self.first = first or last + self.last = last def __rshift__(self, other): """ Self >> Other """ - chain = self.graph.add_chain(other, _input=self.node) - return GraphCursor(chain.graph, chain.output) + + if other == ...: + raise NotImplementedError( + "Expected something looking like a node, but got an Ellipsis (...). Did you forget to complete the graph?" + ) + + nodes = other.nodes if hasattr(other, "nodes") else [other] + + if len(nodes): + chain = self.graph.add_chain(*nodes, _input=self.last) + return GraphCursor(chain.graph, first=self.first, last=chain.output) + + return self + + +class PartialGraph: + def __init__(self, *nodes): + self.nodes = list(nodes) class Graph: """ Represents a directed graph of nodes. """ - name = '' + + name = "" def __init__(self, *chain): self.edges = {BEGIN: set()} @@ -46,7 +73,7 @@ class Graph: return self.nodes[key] def get_cursor(self, ref=BEGIN): - return GraphCursor(self, self._resolve_index(ref)) + return GraphCursor(self, last=self._resolve_index(ref)) def outputs_of(self, idx, create=False): """ Get a set of the outputs for a given node index. @@ -76,7 +103,7 @@ class Graph: _last = self.add_node(node) if not i and _name: if _name in self.named: - raise KeyError('Duplicate name {!r} in graph.'.format(_name)) + raise KeyError("Duplicate name {!r} in graph.".format(_name)) self.named[_name] = _last if _first is None: _first = _last @@ -86,7 +113,7 @@ class Graph: if _output is not None: self.outputs_of(_input, create=True).add(_output) - if hasattr(self, '_topologcally_sorted_indexes_cache'): + if hasattr(self, "_topologcally_sorted_indexes_cache"): del self._topologcally_sorted_indexes_cache return GraphRange(self, _first, _last) @@ -144,10 +171,10 @@ class Graph: return self._graphviz except AttributeError: g = Digraph() - g.attr(rankdir='LR') - g.node('BEGIN', shape='point') + g.attr(rankdir="LR") + g.node("BEGIN", shape="point") for i in self.outputs_of(BEGIN): - g.edge('BEGIN', str(i)) + g.edge("BEGIN", str(i)) for ix in self.topologically_sorted_indexes: g.node(str(ix), label=get_name(self[ix])) for iy in self.outputs_of(ix): @@ -160,9 +187,9 @@ class Graph: def _repr_html_(self): try: - return '
{}
{}
'.format(self.graphviz._repr_svg_(), html.escape(repr(self))) + return "
{}
{}
".format(self.graphviz._repr_svg_(), html.escape(repr(self))) except (ExecutableNotFound, FileNotFoundError) as exc: - return '{}: {}'.format(type(exc).__name__, str(exc)) + return "{}: {}".format(type(exc).__name__, str(exc)) def _resolve_index(self, mixed): """ @@ -182,10 +209,10 @@ class Graph: if mixed in self.nodes: return self.nodes.index(mixed) - raise ValueError('Cannot find node matching {!r}.'.format(mixed)) + raise ValueError("Cannot find node matching {!r}.".format(mixed)) def _get_graphviz_node_id(graph, i): escaped_index = str(i) escaped_name = json.dumps(get_name(graph[i])) - return '{{{} [label={}]}}'.format(escaped_index, escaped_name) + return "{{{} [label={}]}}".format(escaped_index, escaped_name) diff --git a/bonobo/structs/inputs.py b/bonobo/structs/inputs.py index 2731000..f0277b1 100644 --- a/bonobo/structs/inputs.py +++ b/bonobo/structs/inputs.py @@ -70,7 +70,7 @@ class Input(Queue, Readable, Writable): # Check we are actually able to receive data. if self._writable_runlevel < 1: - raise InactiveWritableError('Cannot put() on an inactive {}.'.format(Writable.__name__)) + raise InactiveWritableError("Cannot put() on an inactive {}.".format(Writable.__name__)) if data == END: self._writable_runlevel -= 1 @@ -85,7 +85,7 @@ class Input(Queue, Readable, Writable): def get(self, block=True, timeout=None): if not self.alive: - raise InactiveReadableError('Cannot get() on an inactive {}.'.format(Readable.__name__)) + raise InactiveReadableError("Cannot get() on an inactive {}.".format(Readable.__name__)) data = Queue.get(self, block, timeout) @@ -94,7 +94,7 @@ class Input(Queue, Readable, Writable): if not self.alive: raise InactiveReadableError( - 'Cannot get() on an inactive {} (runlevel just reached 0).'.format(Readable.__name__) + "Cannot get() on an inactive {} (runlevel just reached 0).".format(Readable.__name__) ) return self.get(block, timeout) diff --git a/bonobo/structs/tokens.py b/bonobo/structs/tokens.py index 9b8f151..3edf773 100644 --- a/bonobo/structs/tokens.py +++ b/bonobo/structs/tokens.py @@ -3,7 +3,7 @@ class Token: self.__name__ = name def __repr__(self): - return '<{}>'.format(self.__name__) + return "<{}>".format(self.__name__) class Flag(Token): diff --git a/bonobo/util/__init__.py b/bonobo/util/__init__.py index 894d053..881fada 100644 --- a/bonobo/util/__init__.py +++ b/bonobo/util/__init__.py @@ -16,25 +16,25 @@ from bonobo.util.inspect import ( istuple, istype, ) -from bonobo.util.objects import (get_name, get_attribute_or_create, ValueHolder) +from bonobo.util.objects import get_name, get_attribute_or_create, ValueHolder # Bonobo's util API __all__ = [ - 'ValueHolder', - 'cast', - 'deprecated', - 'deprecated_alias', - 'ensure_tuple', - 'get_attribute_or_create', - 'get_name', - 'inspect_node', - 'isconfigurable', - 'isconfigurabletype', - 'iscontextprocessor', - 'isdict', - 'ismethod', - 'isoption', - 'istype', - 'sortedlist', - 'tuplize', + "ValueHolder", + "cast", + "deprecated", + "deprecated_alias", + "ensure_tuple", + "get_attribute_or_create", + "get_name", + "inspect_node", + "isconfigurable", + "isconfigurabletype", + "iscontextprocessor", + "isdict", + "ismethod", + "isoption", + "istype", + "sortedlist", + "tuplize", ] diff --git a/bonobo/util/api.py b/bonobo/util/api.py index 7bfba02..37754ee 100644 --- a/bonobo/util/api.py +++ b/bonobo/util/api.py @@ -12,14 +12,14 @@ class ApiHelper: if graph: # This function must comply to the "graph" API interface, meaning it can bahave like bonobo.run. from inspect import signature + parameters = list(signature(x).parameters) - required_parameters = {'plugins', 'services', 'strategy'} - assert len(parameters - ) > 0 and parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".' - assert required_parameters.intersection( - parameters - ) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join( - sorted(required_parameters) + required_parameters = {"plugins", "services", "strategy"} + assert ( + len(parameters) > 0 and parameters[0] == "graph" + ), 'First parameter of a graph api function must be "graph".' + assert required_parameters.intersection(parameters) == required_parameters, ( + "Graph api functions must define the following parameters: " + ", ".join(sorted(required_parameters)) ) self.__all__.append(get_name(x)) diff --git a/bonobo/util/bags.py b/bonobo/util/bags.py index fd31d06..c0d221e 100644 --- a/bonobo/util/bags.py +++ b/bonobo/util/bags.py @@ -71,16 +71,18 @@ class {typename}(tuple): {field_defs} ''' -_field_template = '''\ +_field_template = """\ {name} = _property(_itemgetter({index:d}), doc={doc!r}) -'''.strip('\n') - -_reserved = frozenset( - ['_', '_cls', '_attrs', '_fields', 'get', '_asdict', '_replace', '_make', 'self', '_self', 'tuple'] + dir(tuple) +""".strip( + "\n" ) -_multiple_underscores_pattern = re.compile('__+') -_slugify_allowed_chars_pattern = re.compile(r'[^a-z0-9_]+', flags=re.IGNORECASE) +_reserved = frozenset( + ["_", "_cls", "_attrs", "_fields", "get", "_asdict", "_replace", "_make", "self", "_self", "tuple"] + dir(tuple) +) + +_multiple_underscores_pattern = re.compile("__+") +_slugify_allowed_chars_pattern = re.compile(r"[^a-z0-9_]+", flags=re.IGNORECASE) def _uniquify(f): @@ -90,13 +92,13 @@ def _uniquify(f): def _uniquified(x): nonlocal f, seen x = str(x) - v = v0 = _multiple_underscores_pattern.sub('_', f(x)) + v = v0 = _multiple_underscores_pattern.sub("_", f(x)) i = 0 # if last character is not "allowed", let's start appending indexes right from the first iteration if len(x) and _slugify_allowed_chars_pattern.match(x[-1]): - v = '{}{}'.format(v0, i) + v = "{}{}".format(v0, i) while v in seen: - v = '{}{}'.format(v0, i) + v = "{}{}".format(v0, i) i += 1 seen.add(v) return v @@ -106,13 +108,13 @@ def _uniquify(f): def _make_valid_attr_name(x): if iskeyword(x): - x = '_' + x + x = "_" + x if x.isidentifier(): return x - x = slugify(x, separator='_', regex_pattern=_slugify_allowed_chars_pattern) + x = slugify(x, separator="_", regex_pattern=_slugify_allowed_chars_pattern) if x.isidentifier(): return x - x = '_' + x + x = "_" + x if x.isidentifier(): return x raise ValueError(x) @@ -124,23 +126,23 @@ def BagType(typename, fields, *, verbose=False, module=None): attrs = tuple(map(_uniquify(_make_valid_attr_name), fields)) if type(fields) is str: - raise TypeError('BagType does not support providing fields as a string.') + raise TypeError("BagType does not support providing fields as a string.") fields = list(map(str, fields)) typename = str(typename) for i, name in enumerate([typename] + fields): if type(name) is not str: - raise TypeError('Type names and field names must be strings, got {name!r}'.format(name=name)) + raise TypeError("Type names and field names must be strings, got {name!r}".format(name=name)) if not i: if not name.isidentifier(): - raise ValueError('Type names must be valid identifiers: {name!r}'.format(name=name)) + raise ValueError("Type names must be valid identifiers: {name!r}".format(name=name)) if iskeyword(name): - raise ValueError('Type names cannot be a keyword: {name!r}'.format(name=name)) + raise ValueError("Type names cannot be a keyword: {name!r}".format(name=name)) seen = set() for name in fields: if name in seen: - raise ValueError('Encountered duplicate field name: {name!r}'.format(name=name)) + raise ValueError("Encountered duplicate field name: {name!r}".format(name=name)) seen.add(name) # Fill-in the class template @@ -150,21 +152,24 @@ def BagType(typename, fields, *, verbose=False, module=None): attrs=attrs, num_fields=len(fields), arg_list=repr(attrs).replace("'", "")[1:-1], - repr_fmt=', '.join(('%r' if isinstance(fields[index], int) else '{name}=%r').format(name=name) - for index, name in enumerate(attrs)), - field_defs='\n'.join( + repr_fmt=", ".join( + ("%r" if isinstance(fields[index], int) else "{name}=%r").format(name=name) + for index, name in enumerate(attrs) + ), + field_defs="\n".join( _field_template.format( index=index, name=name, - doc='Alias for ' + - ('field #{}'.format(index) if isinstance(fields[index], int) else repr(fields[index])) - ) for index, name in enumerate(attrs) - ) + doc="Alias for " + + ("field #{}".format(index) if isinstance(fields[index], int) else repr(fields[index])), + ) + for index, name in enumerate(attrs) + ), ) # Execute the template string in a temporary namespace and support # tracing utilities by setting a value for frame.f_globals['__name__'] - namespace = dict(__name__='namedtuple_%s' % typename) + namespace = dict(__name__="namedtuple_%s" % typename) exec(class_definition, namespace) result = namespace[typename] result._source = class_definition @@ -178,7 +183,7 @@ def BagType(typename, fields, *, verbose=False, module=None): # specified a particular module. if module is None: try: - module = sys._getframe(1).f_globals.get('__name__', '__main__') + module = sys._getframe(1).f_globals.get("__name__", "__main__") except (AttributeError, ValueError): pass if module is not None: diff --git a/bonobo/util/collections.py b/bonobo/util/collections.py index 1234142..9133e09 100644 --- a/bonobo/util/collections.py +++ b/bonobo/util/collections.py @@ -25,7 +25,7 @@ def _with_length_check(f): if length is not None: if length != len(result): raise TypeError( - 'Length check failed, expected {} fields but got {}: {!r}.'.format(length, len(result), result) + "Length check failed, expected {} fields but got {}: {!r}.".format(length, len(result), result) ) return result @@ -54,7 +54,7 @@ def ensure_tuple(tuple_or_mixed, *, cls=None): if isinstance(tuple_or_mixed, tuple): return tuple.__new__(cls, tuple_or_mixed) - return tuple.__new__(cls, (tuple_or_mixed, )) + return tuple.__new__(cls, (tuple_or_mixed,)) def cast(type_): diff --git a/bonobo/util/compat.py b/bonobo/util/compat.py index 4a62742..56e313f 100644 --- a/bonobo/util/compat.py +++ b/bonobo/util/compat.py @@ -5,13 +5,13 @@ import warnings def deprecated_alias(alias, func): @functools.wraps(func) def new_func(*args, **kwargs): - warnings.simplefilter('always', DeprecationWarning) # turn off filter + warnings.simplefilter("always", DeprecationWarning) # turn off filter warnings.warn( "Call to deprecated function alias {}, use {} instead.".format(alias, func.__name__), category=DeprecationWarning, - stacklevel=2 + stacklevel=2, ) - warnings.simplefilter('default', DeprecationWarning) # reset filter + warnings.simplefilter("default", DeprecationWarning) # reset filter return func(*args, **kwargs) return new_func @@ -24,11 +24,11 @@ def deprecated(func): @functools.wraps(func) def new_func(*args, **kwargs): - warnings.simplefilter('always', DeprecationWarning) # turn off filter + warnings.simplefilter("always", DeprecationWarning) # turn off filter warnings.warn( "Call to deprecated function {}.".format(func.__name__), category=DeprecationWarning, stacklevel=2 ) - warnings.simplefilter('default', DeprecationWarning) # reset filter + warnings.simplefilter("default", DeprecationWarning) # reset filter return func(*args, **kwargs) return new_func diff --git a/bonobo/util/envelopes.py b/bonobo/util/envelopes.py index 04f8080..d888342 100644 --- a/bonobo/util/envelopes.py +++ b/bonobo/util/envelopes.py @@ -1,8 +1,8 @@ from bonobo.structs.tokens import Flag -F_INHERIT = Flag('Inherit') +F_INHERIT = Flag("Inherit") -F_NOT_MODIFIED = Flag('NotModified') +F_NOT_MODIFIED = Flag("NotModified") F_NOT_MODIFIED.must_be_first = True F_NOT_MODIFIED.must_be_last = True F_NOT_MODIFIED.allows_data = False diff --git a/bonobo/util/environ.py b/bonobo/util/environ.py index 980d1db..b1a6635 100644 --- a/bonobo/util/environ.py +++ b/bonobo/util/environ.py @@ -5,12 +5,12 @@ import re import warnings from contextlib import contextmanager -__escape_decoder = codecs.getdecoder('unicode_escape') -__posix_variable = re.compile('\$\{[^\}]*\}') +__escape_decoder = codecs.getdecoder("unicode_escape") +__posix_variable = re.compile("\$\{[^\}]*\}") def parse_var(var): - name, value = var.split('=', 1) + name, value = var.split("=", 1) def decode_escaped(escaped): return __escape_decoder(escaped)[0] @@ -29,15 +29,15 @@ def load_env_from_file(filename): Read an env file into a collection of (name, value) tuples. """ if not os.path.exists(filename): - raise FileNotFoundError('Environment file {} does not exist.'.format(filename)) + raise FileNotFoundError("Environment file {} does not exist.".format(filename)) with open(filename) as f: for lineno, line in enumerate(f): line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - if '=' not in line: - raise SyntaxError('Invalid environment file syntax in {} at line {}.'.format(filename, lineno + 1)) + if "=" not in line: + raise SyntaxError("Invalid environment file syntax in {} at line {}.".format(filename, lineno + 1)) name, value = parse_var(line) @@ -64,10 +64,10 @@ def get_argument_parser(parser=None): global _parser _parser = parser - _parser.add_argument('--default-env-file', '-E', action='append') - _parser.add_argument('--default-env', action='append') - _parser.add_argument('--env-file', action='append') - _parser.add_argument('--env', '-e', action='append') + _parser.add_argument("--default-env-file", "-E", action="append") + _parser.add_argument("--default-env", action="append") + _parser.add_argument("--env-file", action="append") + _parser.add_argument("--env", "-e", action="append") return _parser @@ -89,10 +89,11 @@ def parse_args(mixed=None): global _parser if _parser is not None: warnings.warn( - 'You are calling bonobo.parse_args() without a parser argument, but it looks like you created a parser before. You probably want to pass your parser to this call, or if creating a new parser here is really what you want to do, please create a new one explicitely to silence this warning.' + "You are calling bonobo.parse_args() without a parser argument, but it looks like you created a parser before. You probably want to pass your parser to this call, or if creating a new parser here is really what you want to do, please create a new one explicitely to silence this warning." ) # use the api from bonobo namespace, in case a command patched it. import bonobo + mixed = bonobo.get_argument_parser() if isinstance(mixed, argparse.ArgumentParser): @@ -117,14 +118,14 @@ def parse_args(mixed=None): # env-file sets something.) try: # Set default environment - for name, value in map(parse_var, options.pop('default_env', []) or []): + for name, value in map(parse_var, options.pop("default_env", []) or []): if not name in os.environ: if not name in _backup: _backup[name] = os.environ.get(name, None) os.environ[name] = value # Read and set default environment from file(s) - for filename in options.pop('default_env_file', []) or []: + for filename in options.pop("default_env_file", []) or []: for name, value in load_env_from_file(filename): if not name in os.environ: if not name in _backup: @@ -132,14 +133,14 @@ def parse_args(mixed=None): os.environ[name] = value # Read and set environment from file(s) - for filename in options.pop('env_file', []) or []: + for filename in options.pop("env_file", []) or []: for name, value in load_env_from_file(filename): if not name in _backup: _backup[name] = os.environ.get(name, None) os.environ[name] = value # Set environment - for name, value in map(parse_var, options.pop('env', []) or []): + for name, value in map(parse_var, options.pop("env", []) or []): if not name in _backup: _backup[name] = os.environ.get(name, None) os.environ[name] = value diff --git a/bonobo/util/errors.py b/bonobo/util/errors.py index a24ce94..a14ebdd 100644 --- a/bonobo/util/errors.py +++ b/bonobo/util/errors.py @@ -15,36 +15,32 @@ def sweeten_errors(): except Exception as exc: SPACES = 2 w = term.white - prefix = w('║' + ' ' * (SPACES - 1)) - suffix = w(' ' * (SPACES - 1) + '║') + prefix = w("║" + " " * (SPACES - 1)) + suffix = w(" " * (SPACES - 1) + "║") - pre_re = re.compile('([^`]*)`([^`]*)`([^`]*)') + pre_re = re.compile("([^`]*)`([^`]*)`([^`]*)") def format_arg(arg): - length = len(pre_re.sub('\\1\\2\\3', arg)) + length = len(pre_re.sub("\\1\\2\\3", arg)) - arg = pre_re.sub(w('\\1') + term.bold('\\2') + w('\\3'), arg) - arg = re.sub('^ \$ (.*)', term.lightblack(' $ ') + term.reset('\\1'), arg) + arg = pre_re.sub(w("\\1") + term.bold("\\2") + w("\\3"), arg) + arg = re.sub("^ \$ (.*)", term.lightblack(" $ ") + term.reset("\\1"), arg) return (arg, length) def f(*args): - return ''.join(args) + return "".join(args) term_width, term_height = term.get_size() line_length = min(80, term_width) for arg in exc.args: line_length = max(min(line_length, len(arg) + 2 * SPACES), 120) - print(f(w('╔' + '═' * (line_length - 2) + '╗'))) + print(f(w("╔" + "═" * (line_length - 2) + "╗"))) for i, arg in enumerate(exc.args): if i == 1: - print(f( - prefix, - ' ' * (line_length - 2 * SPACES), - suffix, - )) + print(f(prefix, " " * (line_length - 2 * SPACES), suffix)) arg_formatted, arg_length = format_arg(arg) if not i: @@ -52,17 +48,17 @@ def sweeten_errors(): print( f( prefix, - term.red_bg(term.bold(' ' + type(exc).__name__ + ' ')), - ' ', + term.red_bg(term.bold(" " + type(exc).__name__ + " ")), + " ", w(arg_formatted), - ' ' * (line_length - (arg_length + 3 + len(type(exc).__name__) + 2 * SPACES)), + " " * (line_length - (arg_length + 3 + len(type(exc).__name__) + 2 * SPACES)), suffix, ) ) else: # other lines - print(f(prefix, arg_formatted + ' ' * (line_length - arg_length - 2 * SPACES), suffix)) + print(f(prefix, arg_formatted + " " * (line_length - arg_length - 2 * SPACES), suffix)) - print(f(w('╚' + '═' * (line_length - 2) + '╝'))) + print(f(w("╚" + "═" * (line_length - 2) + "╝"))) - logging.getLogger().debug('This error was caused by the following exception chain.', exc_info=exc_info()) + logging.getLogger().debug("This error was caused by the following exception chain.", exc_info=exc_info()) diff --git a/bonobo/util/inspect.py b/bonobo/util/inspect.py index a7f27c4..da74144 100644 --- a/bonobo/util/inspect.py +++ b/bonobo/util/inspect.py @@ -9,6 +9,7 @@ def isconfigurable(mixed): :return: bool """ from bonobo.config.configurables import Configurable + return isinstance(mixed, Configurable) @@ -32,7 +33,7 @@ def isconfigurabletype(mixed, *, strict=False): if isinstance(mixed, PartiallyConfigured): return True - if hasattr(mixed, '_partial') and mixed._partial: + if hasattr(mixed, "_partial") and mixed._partial: return True return False @@ -47,6 +48,7 @@ def isoption(mixed): """ from bonobo.config.options import Option + return isinstance(mixed, Option) @@ -58,6 +60,7 @@ def ismethod(mixed): :return: bool """ from bonobo.config.options import Method + return isinstance(mixed, Method) @@ -69,6 +72,7 @@ def iscontextprocessor(x): :return: bool """ from bonobo.config.processors import ContextProcessor + return isinstance(x, ContextProcessor) @@ -102,15 +106,7 @@ def istuple(mixed): return isinstance(mixed, tuple) -ConfigurableInspection = namedtuple( - 'ConfigurableInspection', [ - 'type', - 'instance', - 'options', - 'processors', - 'partial', - ] -) +ConfigurableInspection = namedtuple("ConfigurableInspection", ["type", "instance", "options", "processors", "partial"]) ConfigurableInspection.__enter__ = lambda self: self ConfigurableInspection.__exit__ = lambda *exc_details: None @@ -134,17 +130,11 @@ def inspect_node(mixed, *, _partial=None): inst, typ = None, mixed elif isconfigurable(mixed): inst, typ = mixed, type(mixed) - elif hasattr(mixed, 'func'): + elif hasattr(mixed, "func"): return inspect_node(mixed.func, _partial=(mixed.args, mixed.keywords)) else: raise TypeError( - 'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.' + "Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs." ) - return ConfigurableInspection( - typ, - inst, - list(typ.__options__), - list(typ.__processors__), - _partial, - ) + return ConfigurableInspection(typ, inst, list(typ.__options__), list(typ.__processors__), _partial) diff --git a/bonobo/util/objects.py b/bonobo/util/objects.py index f3ffa5e..847b10d 100644 --- a/bonobo/util/objects.py +++ b/bonobo/util/objects.py @@ -11,7 +11,7 @@ class Wrapper: @property def __name__(self): - return getattr(self.wrapped, '__name__', getattr(type(self.wrapped), '__name__', repr(self.wrapped))) + return getattr(self.wrapped, "__name__", getattr(type(self.wrapped), "__name__", repr(self.wrapped))) name = __name__ @@ -142,10 +142,10 @@ class ValueHolder: return divmod(other, self._value) def __pow__(self, other): - return self._value**other + return self._value ** other def __rpow__(self, other): - return other**self._value + return other ** self._value def __ipow__(self, other): self._value **= other diff --git a/bonobo/util/pkgs.py b/bonobo/util/pkgs.py index e4de1b6..fc6ac18 100644 --- a/bonobo/util/pkgs.py +++ b/bonobo/util/pkgs.py @@ -4,5 +4,5 @@ from packaging.utils import canonicalize_name bonobo_packages = {} for p in pkg_resources.working_set: name = canonicalize_name(p.project_name) - if name.startswith('bonobo'): + if name.startswith("bonobo"): bonobo_packages[name] = p diff --git a/bonobo/util/resolvers.py b/bonobo/util/resolvers.py index 5cf2738..f49acad 100644 --- a/bonobo/util/resolvers.py +++ b/bonobo/util/resolvers.py @@ -23,8 +23,8 @@ class _ModulesRegistry(dict): def require(self, name): if name not in self: - bits = name.split('.') - filename = os.path.join(self.pathname, *bits[:-1], bits[-1] + '.py') + bits = name.split(".") + filename = os.path.join(self.pathname, *bits[:-1], bits[-1] + ".py") self[name] = _RequiredModule(runpy.run_path(filename, run_name=name)) return self[name] @@ -37,7 +37,7 @@ def _parse_option(option): :return: tuple """ try: - key, val = option.split('=', 1) + key, val = option.split("=", 1) except ValueError: return option, True @@ -75,7 +75,7 @@ def _resolve_transformations(transformations): transformations = transformations or [] for t in transformations: try: - mod, attr = t.split(':', 1) + mod, attr = t.split(":", 1) yield getattr(registry.require(mod), attr) except ValueError: yield getattr(bonobo, t) diff --git a/bonobo/util/statistics.py b/bonobo/util/statistics.py index dce2714..62a5075 100644 --- a/bonobo/util/statistics.py +++ b/bonobo/util/statistics.py @@ -10,8 +10,8 @@ class WithStatistics: return ((name, self.statistics[name]) for name in self.statistics_names) def get_statistics_as_string(self, *args, **kwargs): - stats = tuple('{0}={1}'.format(name, cnt) for name, cnt in self.get_statistics(*args, **kwargs) if cnt > 0) - return (kwargs.get('prefix', '') + ' '.join(stats)) if len(stats) else '' + stats = tuple("{0}={1}".format(name, cnt) for name, cnt in self.get_statistics(*args, **kwargs) if cnt > 0) + return (kwargs.get("prefix", "") + " ".join(stats)) if len(stats) else "" def increment(self, name, *, amount=1): self.statistics[name] += amount @@ -35,4 +35,4 @@ class Timer: return self.__finish - self.__start def __str__(self): - return str(int(self.duration * 1000) / 1000.0) + 's' + return str(int(self.duration * 1000) / 1000.0) + "s" diff --git a/bonobo/util/term.py b/bonobo/util/term.py index 2fa02ce..389f02f 100644 --- a/bonobo/util/term.py +++ b/bonobo/util/term.py @@ -1,2 +1,2 @@ -CLEAR_EOL = '\033[0K' -MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n) +CLEAR_EOL = "\033[0K" +MOVE_CURSOR_UP = lambda n: "\033[{}A".format(n) diff --git a/bonobo/util/testing.py b/bonobo/util/testing.py index f7b5108..13554f8 100644 --- a/bonobo/util/testing.py +++ b/bonobo/util/testing.py @@ -26,20 +26,20 @@ def optional_contextmanager(cm, *, ignore=False): class FilesystemTester: - def __init__(self, extension='txt', mode='w', *, input_data=''): + def __init__(self, extension="txt", mode="w", *, input_data=""): self.extension = extension self.input_data = input_data self.mode = mode def get_services_for_reader(self, tmpdir): - fs, filename = open_fs(tmpdir), 'input.' + self.extension + fs, filename = open_fs(tmpdir), "input." + self.extension with fs.open(filename, self.mode) as fp: fp.write(self.input_data) - return fs, filename, {'fs': fs} + return fs, filename, {"fs": fs} def get_services_for_writer(self, tmpdir): - fs, filename = open_fs(tmpdir), 'output.' + self.extension - return fs, filename, {'fs': fs} + fs, filename = open_fs(tmpdir), "output." + self.extension + return fs, filename, {"fs": fs} class QueueList(list): @@ -60,7 +60,7 @@ class BufferingContext: return self.buffer def get_buffer_args_as_dicts(self): - return [row._asdict() if hasattr(row, '_asdict') else dict(row) for row in self.buffer] + return [row._asdict() if hasattr(row, "_asdict") else dict(row) for row in self.buffer] class BufferingNodeExecutionContext(BufferingContext, NodeExecutionContext): @@ -106,43 +106,37 @@ def runner_entrypoint(args): @runner def runner_module(args): """ Run bonobo using the bonobo.__main__ file, which is equivalent as doing "python -m bonobo ...".""" - with patch.object(sys, 'argv', ['bonobo', *args]): - return runpy.run_path(__main__.__file__, run_name='__main__') + with patch.object(sys, "argv", ["bonobo", *args]): + return runpy.run_path(__main__.__file__, run_name="__main__") -all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module]) +all_runners = pytest.mark.parametrize("runner", [runner_entrypoint, runner_module]) all_environ_targets = pytest.mark.parametrize( - 'target', [ - (get_examples_path('environ.py'), ), - ( - '-m', - 'bonobo.examples.environ', - ), - ] + "target", [(get_examples_path("environ.py"),), ("-m", "bonobo.examples.environ")] ) @all_runners @all_environ_targets -class EnvironmentTestCase(): +class EnvironmentTestCase: def run_quiet(self, runner, *args): - return runner('run', '--quiet', *args) + return runner("run", "--quiet", *args) def run_environ(self, runner, *args, environ=None): - _environ = {'PATH': '/usr/bin'} + _environ = {"PATH": "/usr/bin"} if environ: _environ.update(environ) - with patch.dict('os.environ', _environ, clear=True): + with patch.dict("os.environ", _environ, clear=True): out, err = self.run_quiet(runner, *args) - assert 'SECRET' not in os.environ - assert 'PASSWORD' not in os.environ - if 'PATH' in _environ: - assert 'PATH' in os.environ - assert os.environ['PATH'] == _environ['PATH'] + assert "SECRET" not in os.environ + assert "PASSWORD" not in os.environ + if "PATH" in _environ: + assert "PATH" in os.environ + assert os.environ["PATH"] == _environ["PATH"] - assert err == '' - return dict(map(lambda line: line.split(' ', 1), filter(None, out.split('\n')))) + assert err == "" + return dict(map(lambda line: line.split(" ", 1), filter(None, out.split("\n")))) class StaticNodeTest: @@ -202,8 +196,8 @@ class ReaderTest(ConfigurableNodeTest): ReaderNodeType = None - extension = 'txt' - input_data = '' + extension = "txt" + input_data = "" @property def NodeType(self): @@ -216,12 +210,12 @@ class ReaderTest(ConfigurableNodeTest): self.tmpdir = tmpdir def get_create_args(self, *args): - return (self.filename, ) + args + return (self.filename,) + args def test_customizable_output_type_transform_not_a_type(self): context = self.NodeExecutionContextType( self.create(*self.get_create_args(), output_type=str.upper, **self.get_create_kwargs()), - services=self.services + services=self.services, ) with pytest.raises(TypeError): context.start() @@ -229,9 +223,9 @@ class ReaderTest(ConfigurableNodeTest): def test_customizable_output_type_transform_not_a_tuple(self): context = self.NodeExecutionContextType( self.create( - *self.get_create_args(), output_type=type('UpperString', (str, ), {}), **self.get_create_kwargs() + *self.get_create_args(), output_type=type("UpperString", (str,), {}), **self.get_create_kwargs() ), - services=self.services + services=self.services, ) with pytest.raises(TypeError): context.start() @@ -242,8 +236,8 @@ class WriterTest(ConfigurableNodeTest): WriterNodeType = None - extension = 'txt' - input_data = '' + extension = "txt" + input_data = "" @property def NodeType(self): @@ -256,7 +250,7 @@ class WriterTest(ConfigurableNodeTest): self.tmpdir = tmpdir def get_create_args(self, *args): - return (self.filename, ) + args + return (self.filename,) + args def readlines(self): with self.fs.open(self.filename) as fp: diff --git a/tests/commands/test_clibasics.py b/tests/commands/test_clibasics.py index 1fc292b..780af9c 100644 --- a/tests/commands/test_clibasics.py +++ b/tests/commands/test_clibasics.py @@ -6,20 +6,14 @@ from bonobo.util.testing import all_runners def test_entrypoint(): commands = {} - for command in pkg_resources.iter_entry_points('bonobo.commands'): + for command in pkg_resources.iter_entry_points("bonobo.commands"): commands[command.name] = command - assert not { - 'convert', - 'init', - 'inspect', - 'run', - 'version', - }.difference(set(commands)) + assert not {"convert", "init", "inspect", "run", "version"}.difference(set(commands)) @all_runners def test_no_command(runner): _, err, exc = runner(catch_errors=True) assert type(exc) == SystemExit - assert 'error: the following arguments are required: command' in err \ No newline at end of file + assert "error: the following arguments are required: command" in err diff --git a/tests/commands/test_convert.py b/tests/commands/test_convert.py index d61d8c5..d6c2f96 100644 --- a/tests/commands/test_convert.py +++ b/tests/commands/test_convert.py @@ -8,10 +8,10 @@ from bonobo.util.testing import all_runners @all_runners def test_convert(runner, tmpdir): - csv_content = 'id;name\n1;Romain' - tmpdir.join('in.csv').write(csv_content) + csv_content = "id;name\n1;Romain" + tmpdir.join("in.csv").write(csv_content) with change_working_directory(tmpdir): - runner('convert', 'in.csv', 'out.csv') + runner("convert", "in.csv", "out.csv") - assert tmpdir.join('out.csv').read().strip() == csv_content + assert tmpdir.join("out.csv").read().strip() == csv_content diff --git a/tests/commands/test_download.py b/tests/commands/test_download.py index 83b0ef4..2591c30 100644 --- a/tests/commands/test_download.py +++ b/tests/commands/test_download.py @@ -9,7 +9,7 @@ from bonobo.util.testing import all_runners @all_runners def test_download_works_for_examples(runner): - expected_bytes = b'hello world' + expected_bytes = b"hello world" class MockResponse(object): def __init__(self): @@ -27,12 +27,13 @@ def test_download_works_for_examples(runner): fout = io.BytesIO() fout.close = lambda: None - with patch('bonobo.commands.download._open_url') as mock_open_url, \ - patch('bonobo.commands.download.open') as mock_open: + with patch("bonobo.commands.download._open_url") as mock_open_url, patch( + "bonobo.commands.download.open" + ) as mock_open: mock_open_url.return_value = MockResponse() mock_open.return_value = fout - runner('download', 'examples/datasets/coffeeshops.txt') - expected_url = EXAMPLES_BASE_URL + 'datasets/coffeeshops.txt' + runner("download", "examples/datasets/coffeeshops.txt") + expected_url = EXAMPLES_BASE_URL + "datasets/coffeeshops.txt" mock_open_url.assert_called_once_with(expected_url) assert fout.getvalue() == expected_bytes @@ -41,4 +42,4 @@ def test_download_works_for_examples(runner): @all_runners def test_download_fails_non_example(runner): with pytest.raises(ValueError): - runner('download', 'something/entirely/different.txt') \ No newline at end of file + runner("download", "something/entirely/different.txt") diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 626f5e8..c7f95d2 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -8,22 +8,22 @@ from bonobo.util.testing import all_runners @all_runners def test_init_file(runner, tmpdir): - target = tmpdir.join('foo.py') + target = tmpdir.join("foo.py") target_filename = str(target) - runner('init', target_filename) + runner("init", target_filename) assert os.path.exists(target_filename) - out, err = runner('run', target_filename) - assert out.replace('\n', ' ').strip() == 'Hello World' + out, err = runner("run", target_filename) + assert out.replace("\n", " ").strip() == "Hello World" assert not err @all_runners -@pytest.mark.parametrize('template', InitCommand.TEMPLATES) +@pytest.mark.parametrize("template", InitCommand.TEMPLATES) def test_init_file_templates(runner, template, tmpdir): - target = tmpdir.join('foo.py') + target = tmpdir.join("foo.py") target_filename = str(target) - runner('init', target_filename) + runner("init", target_filename) assert os.path.exists(target_filename) - out, err = runner('run', target_filename) + out, err = runner("run", target_filename) assert not err diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index 69e4f94..da6666a 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -7,42 +7,42 @@ from bonobo.util.testing import all_runners @all_runners def test_run(runner): - out, err = runner('run', '--quiet', get_examples_path('types/strings.py')) - out = out.split('\n') - assert out[0].startswith('Foo ') - assert out[1].startswith('Bar ') - assert out[2].startswith('Baz ') + out, err = runner("run", "--quiet", get_examples_path("types/strings.py")) + out = out.split("\n") + assert out[0].startswith("Foo ") + assert out[1].startswith("Bar ") + assert out[2].startswith("Baz ") @all_runners def test_run_module(runner): - out, err = runner('run', '--quiet', '-m', 'bonobo.examples.types.strings') - out = out.split('\n') - assert out[0].startswith('Foo ') - assert out[1].startswith('Bar ') - assert out[2].startswith('Baz ') + out, err = runner("run", "--quiet", "-m", "bonobo.examples.types.strings") + out = out.split("\n") + assert out[0].startswith("Foo ") + assert out[1].startswith("Bar ") + assert out[2].startswith("Baz ") @all_runners def test_run_path(runner): - out, err = runner('run', '--quiet', get_examples_path('types')) - out = out.split('\n') - assert out[0].startswith('Foo ') - assert out[1].startswith('Bar ') - assert out[2].startswith('Baz ') + out, err = runner("run", "--quiet", get_examples_path("types")) + out = out.split("\n") + assert out[0].startswith("Foo ") + assert out[1].startswith("Bar ") + assert out[2].startswith("Baz ") @all_runners def test_install_requirements_for_dir(runner): - dirname = get_examples_path('types') - with patch('bonobo.commands.run._install_requirements') as install_mock: - runner('run', '--install', dirname) - install_mock.assert_called_once_with(os.path.join(dirname, 'requirements.txt')) + dirname = get_examples_path("types") + with patch("bonobo.commands.run._install_requirements") as install_mock: + runner("run", "--install", dirname) + install_mock.assert_called_once_with(os.path.join(dirname, "requirements.txt")) @all_runners def test_install_requirements_for_file(runner): - dirname = get_examples_path('types') - with patch('bonobo.commands.run._install_requirements') as install_mock: - runner('run', '--install', os.path.join(dirname, 'strings.py')) - install_mock.assert_called_once_with(os.path.join(dirname, 'requirements.txt')) + dirname = get_examples_path("types") + with patch("bonobo.commands.run._install_requirements") as install_mock: + runner("run", "--install", os.path.join(dirname, "strings.py")) + install_mock.assert_called_once_with(os.path.join(dirname, "requirements.txt")) diff --git a/tests/commands/test_run_environ.py b/tests/commands/test_run_environ.py index 3fee6e1..a7ae113 100644 --- a/tests/commands/test_run_environ.py +++ b/tests/commands/test_run_environ.py @@ -5,103 +5,104 @@ from bonobo.util.testing import EnvironmentTestCase @pytest.fixture def env1(tmpdir): - env_file = tmpdir.join('.env_one') - env_file.write('\n'.join(( - 'SECRET=unknown', - 'PASSWORD=sweet', - 'PATH=first', - ))) + env_file = tmpdir.join(".env_one") + env_file.write("\n".join(("SECRET=unknown", "PASSWORD=sweet", "PATH=first"))) return str(env_file) @pytest.fixture def env2(tmpdir): - env_file = tmpdir.join('.env_two') - env_file.write('\n'.join(( - 'PASSWORD=bitter', - "PATH='second'", - ))) + env_file = tmpdir.join(".env_two") + env_file.write("\n".join(("PASSWORD=bitter", "PATH='second'"))) return str(env_file) class TestDefaultEnvFile(EnvironmentTestCase): def test_run_with_default_env_file(self, runner, target, env1): - env = self.run_environ(runner, *target, '--default-env-file', env1) - assert env.get('SECRET') == 'unknown' - assert env.get('PASSWORD') == 'sweet' - assert env.get('PATH') == '/usr/bin' + env = self.run_environ(runner, *target, "--default-env-file", env1) + assert env.get("SECRET") == "unknown" + assert env.get("PASSWORD") == "sweet" + assert env.get("PATH") == "/usr/bin" def test_run_with_multiple_default_env_files(self, runner, target, env1, env2): - env = self.run_environ(runner, *target, '--default-env-file', env1, '--default-env-file', env2) - assert env.get('SECRET') == 'unknown' - assert env.get('PASSWORD') == 'sweet' - assert env.get('PATH') == '/usr/bin' + env = self.run_environ(runner, *target, "--default-env-file", env1, "--default-env-file", env2) + assert env.get("SECRET") == "unknown" + assert env.get("PASSWORD") == "sweet" + assert env.get("PATH") == "/usr/bin" - env = self.run_environ(runner, *target, '--default-env-file', env2, '--default-env-file', env1) - assert env.get('SECRET') == 'unknown' - assert env.get('PASSWORD') == 'bitter' - assert env.get('PATH') == '/usr/bin' + env = self.run_environ(runner, *target, "--default-env-file", env2, "--default-env-file", env1) + assert env.get("SECRET") == "unknown" + assert env.get("PASSWORD") == "bitter" + assert env.get("PATH") == "/usr/bin" class TestEnvFile(EnvironmentTestCase): def test_run_with_file(self, runner, target, env1): - env = self.run_environ(runner, *target, '--env-file', env1) - assert env.get('SECRET') == 'unknown' - assert env.get('PASSWORD') == 'sweet' - assert env.get('PATH') == 'first' + env = self.run_environ(runner, *target, "--env-file", env1) + assert env.get("SECRET") == "unknown" + assert env.get("PASSWORD") == "sweet" + assert env.get("PATH") == "first" def test_run_with_multiple_files(self, runner, target, env1, env2): - env = self.run_environ(runner, *target, '--env-file', env1, '--env-file', env2) - assert env.get('SECRET') == 'unknown' - assert env.get('PASSWORD') == 'bitter' - assert env.get('PATH') == 'second' + env = self.run_environ(runner, *target, "--env-file", env1, "--env-file", env2) + assert env.get("SECRET") == "unknown" + assert env.get("PASSWORD") == "bitter" + assert env.get("PATH") == "second" - env = self.run_environ(runner, *target, '--env-file', env2, '--env-file', env1) - assert env.get('SECRET') == 'unknown' - assert env.get('PASSWORD') == 'sweet' - assert env.get('PATH') == 'first' + env = self.run_environ(runner, *target, "--env-file", env2, "--env-file", env1) + assert env.get("SECRET") == "unknown" + assert env.get("PASSWORD") == "sweet" + assert env.get("PATH") == "first" class TestEnvFileCombinations(EnvironmentTestCase): def test_run_with_both_env_files(self, runner, target, env1, env2): - env = self.run_environ(runner, *target, '--default-env-file', env1, '--env-file', env2) - assert env.get('SECRET') == 'unknown' - assert env.get('PASSWORD') == 'bitter' - assert env.get('PATH') == 'second' + env = self.run_environ(runner, *target, "--default-env-file", env1, "--env-file", env2) + assert env.get("SECRET") == "unknown" + assert env.get("PASSWORD") == "bitter" + assert env.get("PATH") == "second" def test_run_with_both_env_files_then_overrides(self, runner, target, env1, env2): env = self.run_environ( - runner, *target, '--default-env-file', env1, '--env-file', env2, '--env', 'PASSWORD=mine', '--env', - 'SECRET=s3cr3t' + runner, + *target, + "--default-env-file", + env1, + "--env-file", + env2, + "--env", + "PASSWORD=mine", + "--env", + "SECRET=s3cr3t" ) - assert env.get('SECRET') == 's3cr3t' - assert env.get('PASSWORD') == 'mine' - assert env.get('PATH') == 'second' + assert env.get("SECRET") == "s3cr3t" + assert env.get("PASSWORD") == "mine" + assert env.get("PATH") == "second" class TestEnvVars(EnvironmentTestCase): def test_run_no_env(self, runner, target): - env = self.run_environ(runner, *target, environ={'USER': 'romain'}) - assert env.get('USER') == 'romain' + env = self.run_environ(runner, *target, environ={"USER": "romain"}) + assert env.get("USER") == "romain" def test_run_env(self, runner, target): - env = self.run_environ(runner, *target, '--env', 'USER=serious', environ={'USER': 'romain'}) - assert env.get('USER') == 'serious' + env = self.run_environ(runner, *target, "--env", "USER=serious", environ={"USER": "romain"}) + assert env.get("USER") == "serious" def test_run_env_mixed(self, runner, target): - env = self.run_environ(runner, *target, '--env', 'ONE=1', '--env', 'TWO="2"', environ={'USER': 'romain'}) - assert env.get('USER') == 'romain' - assert env.get('ONE') == '1' - assert env.get('TWO') == '2' + env = self.run_environ(runner, *target, "--env", "ONE=1", "--env", 'TWO="2"', environ={"USER": "romain"}) + assert env.get("USER") == "romain" + assert env.get("ONE") == "1" + assert env.get("TWO") == "2" def test_run_default_env(self, runner, target): - env = self.run_environ(runner, *target, '--default-env', 'USER=clown') - assert env.get('USER') == 'clown' + env = self.run_environ(runner, *target, "--default-env", "USER=clown") + assert env.get("USER") == "clown" - env = self.run_environ(runner, *target, '--default-env', 'USER=clown', environ={'USER': 'romain'}) - assert env.get('USER') == 'romain' + env = self.run_environ(runner, *target, "--default-env", "USER=clown", environ={"USER": "romain"}) + assert env.get("USER") == "romain" env = self.run_environ( - runner, *target, '--env', 'USER=serious', '--default-env', 'USER=clown', environ={'USER': 'romain'} + runner, *target, "--env", "USER=serious", "--default-env", "USER=clown", environ={"USER": "romain"} ) - assert env.get('USER') == 'serious' + assert env.get("USER") == "serious" diff --git a/tests/commands/test_version.py b/tests/commands/test_version.py index 1ee893f..35acae5 100644 --- a/tests/commands/test_version.py +++ b/tests/commands/test_version.py @@ -4,17 +4,17 @@ from bonobo.util.testing import all_runners @all_runners def test_version(runner): - out, err = runner('version') + out, err = runner("version") out = out.strip() - assert out.startswith('bonobo ') + assert out.startswith("bonobo ") assert __version__ in out - out, err = runner('version', '-q') + out, err = runner("version", "-q") out = out.strip() - assert out.startswith('bonobo ') + assert out.startswith("bonobo ") assert __version__ in out - out, err = runner('version', '-qq') + out, err = runner("version", "-qq") out = out.strip() - assert not out.startswith('bonobo ') - assert __version__ in out \ No newline at end of file + assert not out.startswith("bonobo ") + assert __version__ in out diff --git a/tests/config/test_configurables.py b/tests/config/test_configurables.py index bee501e..ef6a1f1 100644 --- a/tests/config/test_configurables.py +++ b/tests/config/test_configurables.py @@ -11,7 +11,7 @@ class NoOptConfigurable(Configurable): class MyConfigurable(Configurable): required_str = Option(str) - default_str = Option(str, default='foo') + default_str = Option(str, default="foo") integer = Option(int, required=False) @@ -20,7 +20,7 @@ class MyHarderConfigurable(MyConfigurable): class MyBetterConfigurable(MyConfigurable): - required_str = Option(str, required=False, default='kaboom') + required_str = Option(str, required=False, default="kaboom") class MyConfigurableUsingPositionalOptions(MyConfigurable): @@ -35,7 +35,7 @@ def test_missing_required_option_error(): with pytest.raises(TypeError) as exc: MyConfigurable(_final=True) - assert exc.match('missing 1 required option:') + assert exc.match("missing 1 required option:") def test_missing_required_options_error(): @@ -44,29 +44,29 @@ def test_missing_required_options_error(): with pytest.raises(TypeError) as exc: MyHarderConfigurable(_final=True) - assert exc.match('missing 2 required options:') + assert exc.match("missing 2 required options:") def test_extraneous_option_error(): with pytest.raises(TypeError) as exc: - MyConfigurable(required_str='foo', hello='world') - assert exc.match('got 1 unexpected option:') + MyConfigurable(required_str="foo", hello="world") + assert exc.match("got 1 unexpected option:") def test_extraneous_options_error(): with pytest.raises(TypeError) as exc: - MyConfigurable(required_str='foo', hello='world', acme='corp') - assert exc.match('got 2 unexpected options:') + MyConfigurable(required_str="foo", hello="world", acme="corp") + assert exc.match("got 2 unexpected options:") def test_defaults(): - o = MyConfigurable(required_str='hello') + o = MyConfigurable(required_str="hello") with inspect_node(o) as ni: assert not ni.partial - assert o.required_str == 'hello' - assert o.default_str == 'foo' + assert o.required_str == "hello" + assert o.default_str == "foo" assert o.integer is None @@ -76,30 +76,30 @@ def test_str_type_factory(): with inspect_node(o) as ni: assert not ni.partial - assert o.required_str == '42' - assert o.default_str == 'foo' + assert o.required_str == "42" + assert o.default_str == "foo" assert o.integer is None def test_int_type_factory(): - o = MyConfigurable(required_str='yo', default_str='bar', integer='42') + o = MyConfigurable(required_str="yo", default_str="bar", integer="42") with inspect_node(o) as ni: assert not ni.partial - assert o.required_str == 'yo' - assert o.default_str == 'bar' + assert o.required_str == "yo" + assert o.default_str == "bar" assert o.integer == 42 def test_bool_type_factory(): - o = MyHarderConfigurable(required_str='yes', also_required='True') + o = MyHarderConfigurable(required_str="yes", also_required="True") with inspect_node(o) as ni: assert not ni.partial - assert o.required_str == 'yes' - assert o.default_str == 'foo' + assert o.required_str == "yes" + assert o.default_str == "foo" assert o.integer is None assert o.also_required is True @@ -110,22 +110,22 @@ def test_option_resolution_order(): with inspect_node(o) as ni: assert not ni.partial - assert o.required_str == 'kaboom' - assert o.default_str == 'foo' + assert o.required_str == "kaboom" + assert o.default_str == "foo" assert o.integer is None def test_option_positional(): - o = MyConfigurableUsingPositionalOptions('1', '2', '3', required_str='hello') + o = MyConfigurableUsingPositionalOptions("1", "2", "3", required_str="hello") with inspect_node(o) as ni: assert not ni.partial - assert o.first == '1' - assert o.second == '2' - assert o.third == '3' - assert o.required_str == 'hello' - assert o.default_str == 'foo' + assert o.first == "1" + assert o.second == "2" + assert o.third == "3" + assert o.required_str == "hello" + assert o.default_str == "foo" assert o.integer is None diff --git a/tests/config/test_methods.py b/tests/config/test_methods.py index b5c3a45..9e87c15 100644 --- a/tests/config/test_methods.py +++ b/tests/config/test_methods.py @@ -50,10 +50,7 @@ def test_define_with_decorator(): calls = [] def my_handler(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) Concrete = MethodBasedConfigurable(my_handler) @@ -64,7 +61,7 @@ def test_define_with_decorator(): assert ci.type == MethodBasedConfigurable assert ci.partial - t = Concrete('foo', bar='baz') + t = Concrete("foo", bar="baz") assert callable(t.handler) assert len(calls) == 0 @@ -75,15 +72,12 @@ def test_define_with_decorator(): def test_late_binding_method_decoration(): calls = [] - @MethodBasedConfigurable(foo='foo') + @MethodBasedConfigurable(foo="foo") def Concrete(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) assert callable(Concrete.handler) - t = Concrete(bar='baz') + t = Concrete(bar="baz") assert callable(t.handler) assert len(calls) == 0 @@ -95,12 +89,9 @@ def test_define_with_argument(): calls = [] def concrete_handler(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) - t = MethodBasedConfigurable(concrete_handler, 'foo', bar='baz') + t = MethodBasedConfigurable(concrete_handler, "foo", bar="baz") assert callable(t.handler) assert len(calls) == 0 t() @@ -112,12 +103,9 @@ def test_define_with_inheritance(): class Inheriting(MethodBasedConfigurable): def handler(self, *args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) - t = Inheriting('foo', bar='baz') + t = Inheriting("foo", bar="baz") assert callable(t.handler) assert len(calls) == 0 t() @@ -132,13 +120,10 @@ def test_inheritance_then_decorate(): @Inheriting def Concrete(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) assert callable(Concrete.handler) - t = Concrete('foo', bar='baz') + t = Concrete("foo", bar="baz") assert callable(t.handler) assert len(calls) == 0 t() diff --git a/tests/config/test_methods_partial.py b/tests/config/test_methods_partial.py index e45f11d..d9d6e35 100644 --- a/tests/config/test_methods_partial.py +++ b/tests/config/test_methods_partial.py @@ -12,11 +12,11 @@ class Bobby(Configurable): @ContextProcessor def think(self, context): - yield 'different' + yield "different" def __call__(self, think, *args, **kwargs): - self.handler('1', *args, **kwargs) - self.handler2('2', *args, **kwargs) + self.handler("1", *args, **kwargs) + self.handler2("2", *args, **kwargs) def test_partial(): @@ -40,7 +40,7 @@ def test_partial(): assert len(ci.options) == 4 assert len(ci.processors) == 1 assert ci.partial - assert ci.partial[0] == (f1, ) + assert ci.partial[0] == (f1,) assert not len(ci.partial[1]) # instanciate a more complete partial instance ... @@ -53,13 +53,10 @@ def test_partial(): assert len(ci.options) == 4 assert len(ci.processors) == 1 assert ci.partial - assert ci.partial[0] == ( - f1, - f2, - ) + assert ci.partial[0] == (f1, f2) assert not len(ci.partial[1]) - c = C('foo') + c = C("foo") with inspect_node(c) as ci: assert ci.type == Bobby diff --git a/tests/config/test_processors.py b/tests/config/test_processors.py index c6b3c9a..0417ada 100644 --- a/tests/config/test_processors.py +++ b/tests/config/test_processors.py @@ -1,7 +1,7 @@ from operator import attrgetter from bonobo.config import Configurable -from bonobo.config.processors import ContextProcessor, resolve_processors, ContextCurrifier, use_context_processor +from bonobo.config.processors import ContextCurrifier, ContextProcessor, resolve_processors, use_context_processor class CP1(Configurable): @@ -11,11 +11,11 @@ class CP1(Configurable): @ContextProcessor def a(self): - yield 'this is A' + yield "this is A" @ContextProcessor def b(self, a): - yield a.upper()[:-1] + 'b' + yield a.upper()[:-1] + "b" def __call__(self, a, b): return a, b @@ -46,20 +46,20 @@ class CP3(CP2): def get_all_processors_names(cls): - return list(map(attrgetter('__name__'), resolve_processors(cls))) + return list(map(attrgetter("__name__"), resolve_processors(cls))) def test_inheritance_and_ordering(): - assert get_all_processors_names(CP1) == ['c', 'a', 'b'] - assert get_all_processors_names(CP2) == ['c', 'a', 'b', 'f', 'e', 'd'] - assert get_all_processors_names(CP3) == ['c', 'a', 'b', 'f', 'e', 'd', 'c', 'b'] + assert get_all_processors_names(CP1) == ["c", "a", "b"] + assert get_all_processors_names(CP2) == ["c", "a", "b", "f", "e", "d"] + assert get_all_processors_names(CP3) == ["c", "a", "b", "f", "e", "d", "c", "b"] def test_setup_teardown(): o = CP1() stack = ContextCurrifier(o) stack.setup() - assert o(*stack.args) == ('this is A', 'THIS IS b') + assert o(*stack.args) == ("this is A", "THIS IS b") stack.teardown() @@ -71,4 +71,4 @@ def test_processors_on_func(): def node(context): pass - assert get_all_processors_names(node) == ['cp'] + assert get_all_processors_names(node) == ["cp"] diff --git a/tests/config/test_services.py b/tests/config/test_services.py index 078832c..ffb815b 100644 --- a/tests/config/test_services.py +++ b/tests/config/test_services.py @@ -4,11 +4,11 @@ import time import pytest from bonobo.config import Configurable, Container, Exclusive, Service, use -from bonobo.config.services import validate_service_name, create_container +from bonobo.config.services import create_container, validate_service_name from bonobo.util import get_name -class PrinterInterface(): +class PrinterInterface: def print(self, *args): raise NotImplementedError() @@ -18,46 +18,43 @@ class ConcretePrinter(PrinterInterface): self.prefix = prefix def print(self, *args): - return ';'.join((self.prefix, *args)) + return ";".join((self.prefix, *args)) -SERVICES = Container( - printer0=ConcretePrinter(prefix='0'), - printer1=ConcretePrinter(prefix='1'), -) +SERVICES = Container(printer0=ConcretePrinter(prefix="0"), printer1=ConcretePrinter(prefix="1")) class MyServiceDependantConfigurable(Configurable): - printer = Service(PrinterInterface, ) + printer = Service(PrinterInterface) def __call__(self, *args, printer: PrinterInterface): return printer.print(*args) def test_service_name_validator(): - assert validate_service_name('foo') == 'foo' - assert validate_service_name('foo.bar') == 'foo.bar' - assert validate_service_name('Foo') == 'Foo' - assert validate_service_name('Foo.Bar') == 'Foo.Bar' - assert validate_service_name('Foo.a0') == 'Foo.a0' + assert validate_service_name("foo") == "foo" + assert validate_service_name("foo.bar") == "foo.bar" + assert validate_service_name("Foo") == "Foo" + assert validate_service_name("Foo.Bar") == "Foo.Bar" + assert validate_service_name("Foo.a0") == "Foo.a0" with pytest.raises(ValueError): - validate_service_name('foo.0') + validate_service_name("foo.0") with pytest.raises(ValueError): - validate_service_name('0.foo') + validate_service_name("0.foo") def test_service_dependency(): - o = MyServiceDependantConfigurable(printer='printer0') + o = MyServiceDependantConfigurable(printer="printer0") - assert o('foo', 'bar', printer=SERVICES.get('printer0')) == '0;foo;bar' - assert o('bar', 'baz', printer=SERVICES.get('printer1')) == '1;bar;baz' - assert o('foo', 'bar', **SERVICES.kwargs_for(o)) == '0;foo;bar' + assert o("foo", "bar", printer=SERVICES.get("printer0")) == "0;foo;bar" + assert o("bar", "baz", printer=SERVICES.get("printer1")) == "1;bar;baz" + assert o("foo", "bar", **SERVICES.kwargs_for(o)) == "0;foo;bar" def test_service_dependency_unavailable(): - o = MyServiceDependantConfigurable(printer='printer2') + o = MyServiceDependantConfigurable(printer="printer2") with pytest.raises(KeyError): SERVICES.kwargs_for(o) @@ -72,15 +69,15 @@ class VCR: def test_exclusive(): vcr = VCR() - vcr.append('hello') + vcr.append("hello") def record(prefix, vcr=vcr): with Exclusive(vcr): for i in range(5): - vcr.append(' '.join((prefix, str(i)))) + vcr.append(" ".join((prefix, str(i)))) time.sleep(0.05) - threads = [threading.Thread(target=record, args=(str(i), )) for i in range(5)] + threads = [threading.Thread(target=record, args=(str(i),)) for i in range(5)] for thread in threads: thread.start() @@ -90,8 +87,32 @@ def test_exclusive(): thread.join() assert vcr.tape == [ - 'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3', - '2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4' + "hello", + "0 0", + "0 1", + "0 2", + "0 3", + "0 4", + "1 0", + "1 1", + "1 2", + "1 3", + "1 4", + "2 0", + "2 1", + "2 2", + "2 3", + "2 4", + "3 0", + "3 1", + "3 2", + "3 3", + "3 4", + "4 0", + "4 1", + "4 2", + "4 3", + "4 4", ] @@ -100,28 +121,25 @@ def test_requires(): services = Container(output=vcr.append) - @use('output') + @use("output") def append(out, x): out(x) svcargs = services.kwargs_for(append) assert len(svcargs) == 1 - assert svcargs['output'] == vcr.append + assert svcargs["output"] == vcr.append -@pytest.mark.parametrize('services', [None, {}]) +@pytest.mark.parametrize("services", [None, {}]) def test_create_container_empty_values(services): c = create_container(services) assert len(c) == 2 - assert 'fs' in c and get_name(c['fs']) == 'OSFS' - assert 'http' in c and get_name(c['http']) == 'requests' + assert "fs" in c and get_name(c["fs"]) == "OSFS" + assert "http" in c and get_name(c["http"]) == "requests" def test_create_container_override(): - c = create_container({ - 'http': 'http', - 'fs': 'fs', - }) + c = create_container({"http": "http", "fs": "fs"}) assert len(c) == 2 - assert 'fs' in c and c['fs'] == 'fs' - assert 'http' in c and c['http'] == 'http' + assert "fs" in c and c["fs"] == "fs" + assert "http" in c and c["http"] == "http" diff --git a/tests/examples/test_example_change_some_fields.py b/tests/examples/test_example_change_some_fields.py index 8fd5ff0..33655c2 100644 --- a/tests/examples/test_example_change_some_fields.py +++ b/tests/examples/test_example_change_some_fields.py @@ -5,22 +5,22 @@ from bonobo.config import use_raw_input from bonobo.execution.contexts import GraphExecutionContext from bonobo.util.bags import BagType -Extracted = namedtuple('Extracted', ['id', 'name', 'value']) -ExtractedBT = BagType('ExtractedBT', ['id', 'name', 'value']) +Extracted = namedtuple("Extracted", ["id", "name", "value"]) +ExtractedBT = BagType("ExtractedBT", ["id", "name", "value"]) def extract_nt(): - yield Extracted(id=1, name='Guido', value='.py') - yield Extracted(id=2, name='Larry', value='.pl') - yield Extracted(id=3, name='Dennis', value='.c') - yield Extracted(id=4, name='Yukihiro', value='.rb') + yield Extracted(id=1, name="Guido", value=".py") + yield Extracted(id=2, name="Larry", value=".pl") + yield Extracted(id=3, name="Dennis", value=".c") + yield Extracted(id=4, name="Yukihiro", value=".rb") def extract_bt(): - yield ExtractedBT(id=1, name='Guido', value='.py') - yield ExtractedBT(id=2, name='Larry', value='.pl') - yield ExtractedBT(id=3, name='Dennis', value='.c') - yield ExtractedBT(id=4, name='Yukihiro', value='.rb') + yield ExtractedBT(id=1, name="Guido", value=".py") + yield ExtractedBT(id=2, name="Larry", value=".pl") + yield ExtractedBT(id=3, name="Dennis", value=".c") + yield ExtractedBT(id=4, name="Yukihiro", value=".rb") def transform_using_args(id, name, value): @@ -53,10 +53,18 @@ def test_execution(): with GraphExecutionContext(graph) as context: context.run_until_complete() - assert result_args == [(2, 'Guido', 'guido.py'), (4, 'Larry', 'larry.pl'), (6, 'Dennis', 'dennis.c'), - (8, 'Yukihiro', 'yukihiro.rb')] + assert result_args == [ + (2, "Guido", "guido.py"), + (4, "Larry", "larry.pl"), + (6, "Dennis", "dennis.c"), + (8, "Yukihiro", "yukihiro.rb"), + ] - assert result_nt == [(1, 'GUIDO', '.py'), (2, 'LARRY', '.pl'), (3, 'DENNIS', '.c'), (4, 'YUKIHIRO', '.rb')] + assert result_nt == [(1, "GUIDO", ".py"), (2, "LARRY", ".pl"), (3, "DENNIS", ".c"), (4, "YUKIHIRO", ".rb")] - assert result_bt == [(2, 'Guido', 'guido.py'), (4, 'Larry', 'larry.pl'), (6, 'Dennis', 'dennis.c'), - (8, 'Yukihiro', 'yukihiro.rb')] + assert result_bt == [ + (2, "Guido", "guido.py"), + (4, "Larry", "larry.pl"), + (6, "Dennis", "dennis.c"), + (8, "Yukihiro", "yukihiro.rb"), + ] diff --git a/tests/execution/contexts/test_execution_contexts_graph.py b/tests/execution/contexts/test_execution_contexts_graph.py index e297c31..155936e 100644 --- a/tests/execution/contexts/test_execution_contexts_graph.py +++ b/tests/execution/contexts/test_execution_contexts_graph.py @@ -1,14 +1,14 @@ from bonobo import Graph -from bonobo.constants import EMPTY, BEGIN, END +from bonobo.constants import BEGIN, EMPTY, END from bonobo.execution.contexts import GraphExecutionContext def raise_an_error(*args, **kwargs): - raise Exception('Careful, man, there\'s a beverage here!') + raise Exception("Careful, man, there's a beverage here!") def raise_an_unrecoverrable_error(*args, **kwargs): - raise Exception('You are entering a world of pain!') + raise Exception("You are entering a world of pain!") def test_lifecycle_of_empty_graph(): diff --git a/tests/execution/contexts/test_execution_contexts_node.py b/tests/execution/contexts/test_execution_contexts_node.py index ca9de41..6b694e4 100644 --- a/tests/execution/contexts/test_execution_contexts_node.py +++ b/tests/execution/contexts/test_execution_contexts_node.py @@ -6,131 +6,131 @@ from bonobo import Graph from bonobo.constants import EMPTY from bonobo.execution.contexts.node import NodeExecutionContext, split_token from bonobo.execution.strategies import NaiveStrategy -from bonobo.util.envelopes import F_NOT_MODIFIED, F_INHERIT -from bonobo.util.testing import BufferingNodeExecutionContext, BufferingGraphExecutionContext +from bonobo.util.envelopes import F_INHERIT, F_NOT_MODIFIED +from bonobo.util.testing import BufferingGraphExecutionContext, BufferingNodeExecutionContext def test_node_string(): def f(): - return 'foo' + return "foo" with BufferingNodeExecutionContext(f) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 1 - assert output[0] == ('foo', ) + assert output[0] == ("foo",) def g(): - yield 'foo' - yield 'bar' + yield "foo" + yield "bar" with BufferingNodeExecutionContext(g) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 2 - assert output[0] == ('foo', ) - assert output[1] == ('bar', ) + assert output[0] == ("foo",) + assert output[1] == ("bar",) def test_node_bytes(): def f(): - return b'foo' + return b"foo" with BufferingNodeExecutionContext(f) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 1 - assert output[0] == (b'foo', ) + assert output[0] == (b"foo",) def g(): - yield b'foo' - yield b'bar' + yield b"foo" + yield b"bar" with BufferingNodeExecutionContext(g) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 2 - assert output[0] == (b'foo', ) - assert output[1] == (b'bar', ) + assert output[0] == (b"foo",) + assert output[1] == (b"bar",) def test_node_dict(): def f(): - return {'id': 1, 'name': 'foo'} + return {"id": 1, "name": "foo"} with BufferingNodeExecutionContext(f) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 1 - assert output[0] == ({'id': 1, 'name': 'foo'}, ) + assert output[0] == ({"id": 1, "name": "foo"},) def g(): - yield {'id': 1, 'name': 'foo'} - yield {'id': 2, 'name': 'bar'} + yield {"id": 1, "name": "foo"} + yield {"id": 2, "name": "bar"} with BufferingNodeExecutionContext(g) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 2 - assert output[0] == ({'id': 1, 'name': 'foo'}, ) - assert output[1] == ({'id': 2, 'name': 'bar'}, ) + assert output[0] == ({"id": 1, "name": "foo"},) + assert output[1] == ({"id": 2, "name": "bar"},) def test_node_dict_chained(): strategy = NaiveStrategy(GraphExecutionContextType=BufferingGraphExecutionContext) def f(): - return {'id': 1, 'name': 'foo'} + return {"id": 1, "name": "foo"} def uppercase_name(values): - return {**values, 'name': values['name'].upper()} + return {**values, "name": values["name"].upper()} graph = Graph(f, uppercase_name) context = strategy.execute(graph) output = context.get_buffer() assert len(output) == 1 - assert output[0] == ({'id': 1, 'name': 'FOO'}, ) + assert output[0] == ({"id": 1, "name": "FOO"},) def g(): - yield {'id': 1, 'name': 'foo'} - yield {'id': 2, 'name': 'bar'} + yield {"id": 1, "name": "foo"} + yield {"id": 2, "name": "bar"} graph = Graph(g, uppercase_name) context = strategy.execute(graph) output = context.get_buffer() assert len(output) == 2 - assert output[0] == ({'id': 1, 'name': 'FOO'}, ) - assert output[1] == ({'id': 2, 'name': 'BAR'}, ) + assert output[0] == ({"id": 1, "name": "FOO"},) + assert output[1] == ({"id": 2, "name": "BAR"},) def test_node_tuple(): def f(): - return 'foo', 'bar' + return "foo", "bar" with BufferingNodeExecutionContext(f) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 1 - assert output[0] == ('foo', 'bar') + assert output[0] == ("foo", "bar") def g(): - yield 'foo', 'bar' - yield 'foo', 'baz' + yield "foo", "bar" + yield "foo", "baz" with BufferingNodeExecutionContext(g) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 2 - assert output[0] == ('foo', 'bar') - assert output[1] == ('foo', 'baz') + assert output[0] == ("foo", "bar") + assert output[1] == ("foo", "baz") def test_node_tuple_chained(): @@ -140,50 +140,50 @@ def test_node_tuple_chained(): return tuple(map(str.upper, args)) def f(): - return 'foo', 'bar' + return "foo", "bar" graph = Graph(f, uppercase) context = strategy.execute(graph) output = context.get_buffer() assert len(output) == 1 - assert output[0] == ('FOO', 'BAR') + assert output[0] == ("FOO", "BAR") def g(): - yield 'foo', 'bar' - yield 'foo', 'baz' + yield "foo", "bar" + yield "foo", "baz" graph = Graph(g, uppercase) context = strategy.execute(graph) output = context.get_buffer() assert len(output) == 2 - assert output[0] == ('FOO', 'BAR') - assert output[1] == ('FOO', 'BAZ') + assert output[0] == ("FOO", "BAR") + assert output[1] == ("FOO", "BAZ") def test_node_tuple_dict(): def f(): - return 'foo', 'bar', {'id': 1} + return "foo", "bar", {"id": 1} with BufferingNodeExecutionContext(f) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 1 - assert output[0] == ('foo', 'bar', {'id': 1}) + assert output[0] == ("foo", "bar", {"id": 1}) def g(): - yield 'foo', 'bar', {'id': 1} - yield 'foo', 'baz', {'id': 2} + yield "foo", "bar", {"id": 1} + yield "foo", "baz", {"id": 2} with BufferingNodeExecutionContext(g) as context: context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 2 - assert output[0] == ('foo', 'bar', {'id': 1}) - assert output[1] == ('foo', 'baz', {'id': 2}) + assert output[0] == ("foo", "bar", {"id": 1}) + assert output[1] == ("foo", "baz", {"id": 2}) def test_node_lifecycle_natural(): @@ -229,9 +229,9 @@ def test_node_lifecycle_with_kill(): def test_split_token(): with pytest.deprecated_call(): - assert split_token(('foo', 'bar')) == (set(), ('foo', 'bar')) + assert split_token(("foo", "bar")) == (set(), ("foo", "bar")) assert split_token(()) == (set(), ()) - assert split_token('') == (set(), ('', )) + assert split_token("") == (set(), ("",)) def test_split_token_duplicate(): @@ -247,17 +247,17 @@ def test_split_token_duplicate(): def test_split_token_not_modified(): with pytest.deprecated_call(): with pytest.raises(ValueError): - split_token((F_NOT_MODIFIED, 'foo', 'bar')) + split_token((F_NOT_MODIFIED, "foo", "bar")) with pytest.raises(ValueError): split_token((F_NOT_MODIFIED, F_INHERIT)) with pytest.raises(ValueError): split_token((F_INHERIT, F_NOT_MODIFIED)) assert split_token(F_NOT_MODIFIED) == ({F_NOT_MODIFIED}, ()) - assert split_token((F_NOT_MODIFIED, )) == ({F_NOT_MODIFIED}, ()) + assert split_token((F_NOT_MODIFIED,)) == ({F_NOT_MODIFIED}, ()) def test_split_token_inherit(): with pytest.deprecated_call(): assert split_token(F_INHERIT) == ({F_INHERIT}, ()) - assert split_token((F_INHERIT, )) == ({F_INHERIT}, ()) - assert split_token((F_INHERIT, 'foo', 'bar')) == ({F_INHERIT}, ('foo', 'bar')) + assert split_token((F_INHERIT,)) == ({F_INHERIT}, ()) + assert split_token((F_INHERIT, "foo", "bar")) == ({F_INHERIT}, ("foo", "bar")) diff --git a/tests/execution/test_events.py b/tests/execution/test_events.py index 6fbc405..0c94299 100644 --- a/tests/execution/test_events.py +++ b/tests/execution/test_events.py @@ -6,9 +6,9 @@ from bonobo.execution import events def test_names(): # This test looks useless, but as it's becoming the pliugin API, I want to make sure that nothing changes here, or # notice it otherwise. - for name in 'start', 'started', 'tick', 'stop', 'stopped', 'kill': + for name in "start", "started", "tick", "stop", "stopped", "kill": event_name = getattr(events, name.upper()) - assert event_name == '.'.join(('execution', name)) + assert event_name == ".".join(("execution", name)) def test_event_object(): diff --git a/tests/ext/test_ods.py b/tests/ext/test_ods.py index a07ae81..65ff180 100644 --- a/tests/ext/test_ods.py +++ b/tests/ext/test_ods.py @@ -14,16 +14,11 @@ class ResponseMock: return {} else: self.count += 1 - return { - 'records': self.json_value, - } + return {"records": self.json_value} def test_read_from_opendatasoft_api(): - extract = OpenDataSoftAPI(dataset='test-a-set') - with patch('requests.get', return_value=ResponseMock([ - {'fields': {'foo': 'bar'}}, - {'fields': {'foo': 'zab'}}, - ])): - for line in extract('http://example.com/', ValueHolder(0)): - assert 'foo' in line + extract = OpenDataSoftAPI(dataset="test-a-set") + with patch("requests.get", return_value=ResponseMock([{"fields": {"foo": "bar"}}, {"fields": {"foo": "zab"}}])): + for line in extract("http://example.com/", ValueHolder(0)): + assert "foo" in line diff --git a/tests/features/test_inherit.py b/tests/features/test_inherit.py index 63c6a5d..7b4db7d 100644 --- a/tests/features/test_inherit.py +++ b/tests/features/test_inherit.py @@ -1,27 +1,24 @@ from bonobo.util.envelopes import AppendingEnvelope from bonobo.util.testing import BufferingNodeExecutionContext -messages = [ - ('Hello', ), - ('Goodbye', ), -] +messages = [("Hello",), ("Goodbye",)] def append(*args): - return AppendingEnvelope('!') + return AppendingEnvelope("!") def test_inherit(): with BufferingNodeExecutionContext(append) as context: context.write_sync(*messages) - assert context.get_buffer() == list(map(lambda x: x + ('!', ), messages)) + assert context.get_buffer() == list(map(lambda x: x + ("!",), messages)) def test_inherit_bag_tuple(): with BufferingNodeExecutionContext(append) as context: - context.set_input_fields(['message']) + context.set_input_fields(["message"]) context.write_sync(*messages) - assert context.get_output_fields() == ('message', '0') - assert context.get_buffer() == list(map(lambda x: x + ('!', ), messages)) + assert context.get_output_fields() == ("message", "0") + assert context.get_buffer() == list(map(lambda x: x + ("!",), messages)) diff --git a/tests/features/test_not_modified.py b/tests/features/test_not_modified.py index 63b27d8..8d753da 100644 --- a/tests/features/test_not_modified.py +++ b/tests/features/test_not_modified.py @@ -7,10 +7,7 @@ def useless(*args, **kwargs): def test_not_modified(): - input_messages = [ - ('foo', 'bar'), - ('foo', 'baz'), - ] + input_messages = [("foo", "bar"), ("foo", "baz")] with BufferingNodeExecutionContext(useless) as context: context.write_sync(*input_messages) diff --git a/tests/nodes/io/test_csv.py b/tests/nodes/io/test_csv.py index f862ca7..3173768 100644 --- a/tests/nodes/io/test_csv.py +++ b/tests/nodes/io/test_csv.py @@ -6,13 +6,14 @@ import pytest from bonobo import CsvReader, CsvWriter from bonobo.constants import EMPTY -from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext, WriterTest, ConfigurableNodeTest, \ - ReaderTest +from bonobo.util.testing import ( + BufferingNodeExecutionContext, ConfigurableNodeTest, FilesystemTester, ReaderTest, WriterTest +) -csv_tester = FilesystemTester('csv') -csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar' +csv_tester = FilesystemTester("csv") +csv_tester.input_data = "a,b,c\na foo,b foo,c foo\na bar,b bar,c bar" -defaults = {'lineterminator': '\n'} +defaults = {"lineterminator": "\n"} incontext = ConfigurableNodeTest.incontext @@ -23,15 +24,10 @@ def test_read_csv_from_file_kwargs(tmpdir): with BufferingNodeExecutionContext(CsvReader(filename, **defaults), services=services) as context: context.write_sync(EMPTY) - assert context.get_buffer_args_as_dicts() == [{ - 'a': 'a foo', - 'b': 'b foo', - 'c': 'c foo', - }, { - 'a': 'a bar', - 'b': 'b bar', - 'c': 'c bar', - }] + assert context.get_buffer_args_as_dicts() == [ + {"a": "a foo", "b": "b foo", "c": "c foo"}, + {"a": "a bar", "b": "b bar", "c": "c bar"}, + ] ### @@ -40,86 +36,66 @@ def test_read_csv_from_file_kwargs(tmpdir): class Csv: - extension = 'csv' + extension = "csv" ReaderNodeType = CsvReader WriterNodeType = CsvWriter -L1, L2, L3, L4 = ('a', 'hey'), ('b', 'bee'), ('c', 'see'), ('d', 'dee') -LL = ('i', 'have', 'more', 'values') +L1, L2, L3, L4 = ("a", "hey"), ("b", "bee"), ("c", "see"), ("d", "dee") +LL = ("i", "have", "more", "values") class CsvReaderTest(Csv, ReaderTest, TestCase): - input_data = '\n'.join(( - 'id,name', - '1,John Doe', - '2,Jane Doe', - ',DPR', - '42,Elon Musk', - )) + input_data = "\n".join(("id,name", "1,John Doe", "2,Jane Doe", ",DPR", "42,Elon Musk")) def check_output(self, context, *, prepend=None): out = context.get_buffer() - assert out == (prepend or list()) + [ - ('1', 'John Doe'), - ('2', 'Jane Doe'), - ('', 'DPR'), - ('42', 'Elon Musk'), - ] + assert out == (prepend or list()) + [("1", "John Doe"), ("2", "Jane Doe"), ("", "DPR"), ("42", "Elon Musk")] @incontext() def test_nofields(self, context): context.write_sync(EMPTY) context.stop() self.check_output(context) - assert context.get_output_fields() == ('id', 'name') + assert context.get_output_fields() == ("id", "name") @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - self.check_output(context, prepend=[('id', 'name')]) + self.check_output(context, prepend=[("id", "name")]) - @incontext( - output_fields=( - 'x', - 'y', - ), skip=1 - ) + @incontext(output_fields=("x", "y"), skip=1) def test_output_fields(self, context): context.write_sync(EMPTY) context.stop() self.check_output(context) - assert context.get_output_fields() == ('x', 'y') + assert context.get_output_fields() == ("x", "y") @incontext(quoting=QUOTE_ALL) def test_quoting(self, context): context.write_sync(EMPTY) context.stop() self.check_output(context) - assert context.get_output_fields() == ('id', 'name') + assert context.get_output_fields() == ("id", "name") class CsvWriterTest(Csv, WriterTest, TestCase): @incontext() def test_fields(self, context): - context.set_input_fields(['foo', 'bar']) - context.write_sync(('a', 'b'), ('c', 'd')) + context.set_input_fields(["foo", "bar"]) + context.write_sync(("a", "b"), ("c", "d")) context.stop() - assert self.readlines() == ( - 'foo,bar', - 'a,b', - 'c,d', - ) + assert self.readlines() == ("foo,bar", "a,b", "c,d") @incontext() def test_fields_from_type(self, context): - context.set_input_type(namedtuple('Point', 'x y')) + context.set_input_type(namedtuple("Point", "x y")) context.write_sync((1, 2), (3, 4)) context.stop() - assert self.readlines() == ('x,y', '1,2', '3,4') + assert self.readlines() == ("x,y", "1,2", "3,4") @incontext() def test_nofields_multiple_args(self, context): @@ -127,30 +103,21 @@ class CsvWriterTest(Csv, WriterTest, TestCase): context.write_sync((L1, L2), (L3, L4)) context.stop() - assert self.readlines() == ( - 'a,hey', - 'b,bee', - 'c,see', - 'd,dee', - ) + assert self.readlines() == ("a,hey", "b,bee", "c,see", "d,dee") @incontext() def test_nofields_multiple_args_length_mismatch(self, context): # if length of input vary, then we get a TypeError (unrecoverable) with pytest.raises(TypeError): - context.write_sync((L1, L2), (L3, )) + context.write_sync((L1, L2), (L3,)) @incontext() def test_nofields_single_arg(self, context): # single args are just dumped, shapes can vary. - context.write_sync((L1, ), (LL, ), (L3, )) + context.write_sync((L1,), (LL,), (L3,)) context.stop() - assert self.readlines() == ( - 'a,hey', - 'i,have,more,values', - 'c,see', - ) + assert self.readlines() == ("a,hey", "i,have,more,values", "c,see") @incontext() def test_nofields_empty_args(self, context): diff --git a/tests/nodes/io/test_file.py b/tests/nodes/io/test_file.py index ee8548b..a3e3172 100644 --- a/tests/nodes/io/test_file.py +++ b/tests/nodes/io/test_file.py @@ -5,26 +5,23 @@ from bonobo.constants import EMPTY from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester -txt_tester = FilesystemTester('txt') -txt_tester.input_data = 'Hello\nWorld\n' +txt_tester = FilesystemTester("txt") +txt_tester.input_data = "Hello\nWorld\n" def test_file_writer_contextless(tmpdir): fs, filename, services = txt_tester.get_services_for_writer(tmpdir) with FileWriter(path=filename).open(fs) as fp: - fp.write('Yosh!') + fp.write("Yosh!") with fs.open(filename) as fp: - assert fp.read() == 'Yosh!' + assert fp.read() == "Yosh!" @pytest.mark.parametrize( - 'lines,output', - [ - (('ACME', ), 'ACME'), # one line... - (('Foo', 'Bar', 'Baz'), 'Foo\nBar\nBaz'), # more than one line... - ] + "lines,output", + [(("ACME",), "ACME"), (("Foo", "Bar", "Baz"), "Foo\nBar\nBaz")], # one line... # more than one line... ) def test_file_writer_in_context(tmpdir, lines, output): fs, filename, services = txt_tester.get_services_for_writer(tmpdir) @@ -44,5 +41,5 @@ def test_file_reader(tmpdir): output = context.get_buffer() assert len(output) == 2 - assert output[0] == ('Hello', ) - assert output[1] == ('World', ) + assert output[0] == ("Hello",) + assert output[1] == ("World",) diff --git a/tests/nodes/io/test_json.py b/tests/nodes/io/test_json.py index c2caedd..fac797f 100644 --- a/tests/nodes/io/test_json.py +++ b/tests/nodes/io/test_json.py @@ -4,14 +4,13 @@ from unittest import TestCase import pytest -from bonobo import JsonReader, JsonWriter -from bonobo import LdjsonReader, LdjsonWriter +from bonobo import JsonReader, JsonWriter, LdjsonReader, LdjsonWriter from bonobo.constants import EMPTY -from bonobo.util.testing import WriterTest, ReaderTest, ConfigurableNodeTest +from bonobo.util.testing import ConfigurableNodeTest, ReaderTest, WriterTest -FOOBAR = {'foo': 'bar'} -OD_ABC = OrderedDict((('a', 'A'), ('b', 'B'), ('c', 'C'))) -FOOBAZ = {'foo': 'baz'} +FOOBAR = {"foo": "bar"} +OD_ABC = OrderedDict((("a", "A"), ("b", "B"), ("c", "C"))) +FOOBAZ = {"foo": "baz"} incontext = ConfigurableNodeTest.incontext @@ -21,7 +20,7 @@ incontext = ConfigurableNodeTest.incontext class Json: - extension = 'json' + extension = "json" ReaderNodeType = JsonReader WriterNodeType = JsonWriter @@ -34,88 +33,61 @@ class JsonReaderDictsTest(Json, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ({ - "foo": "bar" - }, ), - ({ - "baz": "boz" - }, ), - ] + assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)] class JsonReaderListsTest(Json, ReaderTest, TestCase): - input_data = '[[1,2,3],\n[4,5,6]]' + input_data = "[[1,2,3],\n[4,5,6]]" @incontext() def test_nofields(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] class JsonReaderStringsTest(Json, ReaderTest, TestCase): - input_data = '[' + ',\n'.join(map(json.dumps, ('foo', 'bar', 'baz'))) + ']' + input_data = "[" + ",\n".join(map(json.dumps, ("foo", "bar", "baz"))) + "]" @incontext() def test_nofields(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [("foo",), ("bar",), ("baz",)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [("foo",), ("bar",), ("baz",)] class JsonWriterTest(Json, WriterTest, TestCase): @incontext() def test_fields(self, context): - context.set_input_fields(['foo', 'bar']) - context.write_sync(('a', 'b'), ('c', 'd')) + context.set_input_fields(["foo", "bar"]) + context.write_sync(("a", "b"), ("c", "d")) context.stop() - assert self.readlines() == ( - '[{"foo": "a", "bar": "b"},', - '{"foo": "c", "bar": "d"}]', - ) + assert self.readlines() == ('[{"foo": "a", "bar": "b"},', '{"foo": "c", "bar": "d"}]') @incontext() def test_fields_from_type(self, context): - context.set_input_type(namedtuple('Point', 'x y')) + context.set_input_type(namedtuple("Point", "x y")) context.write_sync((1, 2), (3, 4)) context.stop() - assert self.readlines() == ( - '[{"x": 1, "y": 2},', - '{"x": 3, "y": 4}]', - ) + assert self.readlines() == ('[{"x": 1, "y": 2},', '{"x": 3, "y": 4}]') @incontext() def test_nofields_multiple_args(self, context): @@ -144,11 +116,7 @@ class JsonWriterTest(Json, WriterTest, TestCase): context.write_sync(FOOBAR, OD_ABC, FOOBAZ) context.stop() - assert self.readlines() == ( - '[{"foo": "bar"},', - '{"a": "A", "b": "B", "c": "C"},', - '{"foo": "baz"}]', - ) + assert self.readlines() == ('[{"foo": "bar"},', '{"a": "A", "b": "B", "c": "C"},', '{"foo": "baz"}]') @incontext() def test_nofields_empty_args(self, context): @@ -156,7 +124,7 @@ class JsonWriterTest(Json, WriterTest, TestCase): context.write_sync(EMPTY, EMPTY, EMPTY) context.stop() - assert self.readlines() == ('[]', ) + assert self.readlines() == ("[]",) ### @@ -165,7 +133,7 @@ class JsonWriterTest(Json, WriterTest, TestCase): class Ldjson: - extension = 'ldjson' + extension = "ldjson" ReaderNodeType = LdjsonReader WriterNodeType = LdjsonWriter @@ -178,85 +146,61 @@ class LdjsonReaderDictsTest(Ldjson, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ({ - "foo": "bar" - }, ), - ({ - "baz": "boz" - }, ), - ] + assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)] class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase): - input_data = '[1,2,3]\n[4,5,6]' + input_data = "[1,2,3]\n[4,5,6]" @incontext() def test_nofields(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase): - input_data = '\n'.join(map(json.dumps, ('foo', 'bar', 'baz'))) + input_data = "\n".join(map(json.dumps, ("foo", "bar", "baz"))) @incontext() def test_nofields(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [("foo",), ("bar",), ("baz",)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [("foo",), ("bar",), ("baz",)] class LdjsonWriterTest(Ldjson, WriterTest, TestCase): @incontext() def test_fields(self, context): - context.set_input_fields(['foo', 'bar']) - context.write_sync(('a', 'b'), ('c', 'd')) + context.set_input_fields(["foo", "bar"]) + context.write_sync(("a", "b"), ("c", "d")) context.stop() assert self.readlines() == ('{"foo": "a", "bar": "b"}', '{"foo": "c", "bar": "d"}') @incontext() def test_fields_from_type(self, context): - context.set_input_type(namedtuple('Point', 'x y')) + context.set_input_type(namedtuple("Point", "x y")) context.write_sync((1, 2), (3, 4)) context.stop() - assert self.readlines() == ( - '{"x": 1, "y": 2}', - '{"x": 3, "y": 4}', - ) + assert self.readlines() == ('{"x": 1, "y": 2}', '{"x": 3, "y": 4}') @incontext() def test_nofields_multiple_args(self, context): @@ -285,11 +229,7 @@ class LdjsonWriterTest(Ldjson, WriterTest, TestCase): context.write_sync(FOOBAR, OD_ABC, FOOBAZ) context.stop() - assert self.readlines() == ( - '{"foo": "bar"}', - '{"a": "A", "b": "B", "c": "C"}', - '{"foo": "baz"}', - ) + assert self.readlines() == ('{"foo": "bar"}', '{"a": "A", "b": "B", "c": "C"}', '{"foo": "baz"}') @incontext() def test_nofields_empty_args(self, context): diff --git a/tests/nodes/io/test_pickle.py b/tests/nodes/io/test_pickle.py index 0662848..3e4745d 100644 --- a/tests/nodes/io/test_pickle.py +++ b/tests/nodes/io/test_pickle.py @@ -7,21 +7,21 @@ from bonobo.constants import EMPTY from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester -pickle_tester = FilesystemTester('pkl', mode='wb') -pickle_tester.input_data = pickle.dumps([['a', 'b', 'c'], ['a foo', 'b foo', 'c foo'], ['a bar', 'b bar', 'c bar']]) +pickle_tester = FilesystemTester("pkl", mode="wb") +pickle_tester.input_data = pickle.dumps([["a", "b", "c"], ["a foo", "b foo", "c foo"], ["a bar", "b bar", "c bar"]]) def test_write_pickled_dict_to_file(tmpdir): fs, filename, services = pickle_tester.get_services_for_writer(tmpdir) with NodeExecutionContext(PickleWriter(filename), services=services) as context: - context.write_sync({'foo': 'bar'}, {'foo': 'baz', 'ignore': 'this'}) + context.write_sync({"foo": "bar"}, {"foo": "baz", "ignore": "this"}) - with fs.open(filename, 'rb') as fp: - assert pickle.loads(fp.read()) == {'foo': 'bar'} + with fs.open(filename, "rb") as fp: + assert pickle.loads(fp.read()) == {"foo": "bar"} with pytest.raises(AttributeError): - getattr(context, 'file') + getattr(context, "file") def test_read_pickled_list_from_file(tmpdir): @@ -31,8 +31,5 @@ def test_read_pickled_list_from_file(tmpdir): context.write_sync(EMPTY) output = context.get_buffer() - assert context.get_output_fields() == ('a', 'b', 'c') - assert output == [ - ('a foo', 'b foo', 'c foo'), - ('a bar', 'b bar', 'c bar'), - ] + assert context.get_output_fields() == ("a", "b", "c") + assert output == [("a foo", "b foo", "c foo"), ("a bar", "b bar", "c bar")] diff --git a/tests/nodes/test_basics.py b/tests/nodes/test_basics.py index 4b6a8be..d977653 100644 --- a/tests/nodes/test_basics.py +++ b/tests/nodes/test_basics.py @@ -5,9 +5,9 @@ from unittest.mock import MagicMock import pytest import bonobo -from bonobo.constants import NOT_MODIFIED, EMPTY -from bonobo.util import ensure_tuple, ValueHolder -from bonobo.util.testing import BufferingNodeExecutionContext, StaticNodeTest, ConfigurableNodeTest +from bonobo.constants import EMPTY, NOT_MODIFIED +from bonobo.util import ValueHolder, ensure_tuple +from bonobo.util.testing import BufferingNodeExecutionContext, ConfigurableNodeTest, StaticNodeTest class CountTest(StaticNodeTest, TestCase): @@ -26,7 +26,7 @@ class CountTest(StaticNodeTest, TestCase): def test_execution(self): with self.execute() as context: context.write_sync(*([EMPTY] * 42)) - assert context.get_buffer() == [(42, )] + assert context.get_buffer() == [(42,)] class IdentityTest(StaticNodeTest, TestCase): @@ -81,14 +81,14 @@ def test_tee(): tee = bonobo.Tee(inner) results = [] for i in range(10): - results.append(tee('foo')) + results.append(tee("foo")) assert results == [NOT_MODIFIED] * 10 assert len(inner.mock_calls) == 10 def test_noop(): - assert bonobo.noop(1, 2, 3, 4, foo='bar') == NOT_MODIFIED + assert bonobo.noop(1, 2, 3, 4, foo="bar") == NOT_MODIFIED def test_fixedwindow(): @@ -98,21 +98,18 @@ def test_fixedwindow(): with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context: context.write_sync(*range(9)) - assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), ( - 8, - None, - )] + assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), (8, None)] with BufferingNodeExecutionContext(bonobo.FixedWindow(1)) as context: context.write_sync(*range(3)) - assert context.get_buffer() == [(0, ), (1, ), (2, )] + assert context.get_buffer() == [(0,), (1,), (2,)] def test_methodcaller(): - with BufferingNodeExecutionContext(methodcaller('swapcase')) as context: - context.write_sync('aaa', 'bBb', 'CcC') - assert context.get_buffer() == list(map(ensure_tuple, ['AAA', 'BbB', 'cCc'])) + with BufferingNodeExecutionContext(methodcaller("swapcase")) as context: + context.write_sync("aaa", "bBb", "CcC") + assert context.get_buffer() == list(map(ensure_tuple, ["AAA", "BbB", "cCc"])) - with BufferingNodeExecutionContext(methodcaller('zfill', 5)) as context: - context.write_sync('a', 'bb', 'ccc') - assert context.get_buffer() == list(map(ensure_tuple, ['0000a', '000bb', '00ccc'])) + with BufferingNodeExecutionContext(methodcaller("zfill", 5)) as context: + context.write_sync("a", "bb", "ccc") + assert context.get_buffer() == list(map(ensure_tuple, ["0000a", "000bb", "00ccc"])) diff --git a/tests/nodes/test_casts.py b/tests/nodes/test_casts.py index 3eb3621..e8696eb 100644 --- a/tests/nodes/test_casts.py +++ b/tests/nodes/test_casts.py @@ -8,11 +8,11 @@ from bonobo.util.bags import BagType from bonobo.util.envelopes import Envelope from bonobo.util.testing import BufferingNodeExecutionContext -MyTuple = namedtuple('MyTuple', ['a', 'b', 'c']) -MyBag = BagType('MyBag', ['a', 'b', 'c']) +MyTuple = namedtuple("MyTuple", ["a", "b", "c"]) +MyBag = BagType("MyBag", ["a", "b", "c"]) -class MyCustomType(): +class MyCustomType: def __init__(self, *args): self.args = args @@ -20,16 +20,19 @@ class MyCustomType(): return MyBag(*self.args) -@pytest.mark.parametrize(['factory', 'expected', 'expected_item0'], [ - [lambda: (1, 2, 3), tuple, int], - [lambda: Envelope((1, 2, 3)), tuple, int], - [lambda: MyTuple(1, 2, 3), MyTuple, int], - [lambda: Envelope(MyTuple(1, 2, 3)), MyTuple, int], - [lambda: MyBag(1, 2, 3), MyBag, int], - [lambda: Envelope(MyBag(1, 2, 3)), MyBag, int], - [lambda: MyCustomType(1, 2, 3), tuple, MyCustomType], - [lambda: Envelope(MyCustomType(1, 2, 3)), tuple, MyCustomType], -]) +@pytest.mark.parametrize( + ["factory", "expected", "expected_item0"], + [ + [lambda: (1, 2, 3), tuple, int], + [lambda: Envelope((1, 2, 3)), tuple, int], + [lambda: MyTuple(1, 2, 3), MyTuple, int], + [lambda: Envelope(MyTuple(1, 2, 3)), MyTuple, int], + [lambda: MyBag(1, 2, 3), MyBag, int], + [lambda: Envelope(MyBag(1, 2, 3)), MyBag, int], + [lambda: MyCustomType(1, 2, 3), tuple, MyCustomType], + [lambda: Envelope(MyCustomType(1, 2, 3)), tuple, MyCustomType], + ], +) def test_casts_after_output(factory: Callable, expected, expected_item0): def transform(): yield factory() diff --git a/tests/plugins/test_console.py b/tests/plugins/test_console.py index 543d341..3a7eb3b 100644 --- a/tests/plugins/test_console.py +++ b/tests/plugins/test_console.py @@ -1,10 +1,11 @@ from unittest.mock import MagicMock +from whistle import EventDispatcher + import bonobo from bonobo.execution import events from bonobo.execution.contexts.graph import GraphExecutionContext from bonobo.plugins.console import ConsoleOutputPlugin -from whistle import EventDispatcher def test_register_unregister(): diff --git a/tests/structs/test_graphs.py b/tests/structs/test_graphs.py index 51321ae..fbca00c 100644 --- a/tests/structs/test_graphs.py +++ b/tests/structs/test_graphs.py @@ -1,7 +1,7 @@ -import pytest - from unittest.mock import sentinel +import pytest + from bonobo.constants import BEGIN from bonobo.structs.graphs import Graph @@ -48,24 +48,14 @@ def test_graph_add_chain(): def test_graph_topological_sort(): g = Graph() - g.add_chain( - sentinel.a1, - sentinel.a2, - sentinel.a3, - _input=None, - _output=None, - ) + g.add_chain(sentinel.a1, sentinel.a2, sentinel.a3, _input=None, _output=None) assert g.topologically_sorted_indexes == (0, 1, 2) assert g[0] == sentinel.a1 assert g[1] == sentinel.a2 assert g[2] == sentinel.a3 - g.add_chain( - sentinel.b1, - sentinel.b2, - _output=sentinel.a2, - ) + g.add_chain(sentinel.b1, sentinel.b2, _output=sentinel.a2) assert g.topologically_sorted_indexes[-2:] == (1, 2) assert g.topologically_sorted_indexes.index(3) < g.topologically_sorted_indexes.index(4) diff --git a/tests/structs/test_inputs.py b/tests/structs/test_inputs.py index d2ce827..c398ca5 100644 --- a/tests/structs/test_inputs.py +++ b/tests/structs/test_inputs.py @@ -19,7 +19,7 @@ from queue import Empty import pytest from bonobo.constants import BEGIN, END -from bonobo.errors import InactiveWritableError, InactiveReadableError +from bonobo.errors import InactiveReadableError, InactiveWritableError from bonobo.structs.inputs import Input @@ -29,22 +29,22 @@ def test_input_runlevels(): # Before BEGIN, noone should be able to write in an Input queue. assert not q.alive with pytest.raises(InactiveWritableError): - q.put('hello, unborn queue.') + q.put("hello, unborn queue.") # Begin q.put(BEGIN) assert q.alive and q._runlevel == 1 - q.put('foo') + q.put("foo") # Second Begin q.put(BEGIN) assert q.alive and q._runlevel == 2 - q.put('bar') + q.put("bar") q.put(END) # FIFO - assert q.get() == 'foo' - assert q.get() == 'bar' + assert q.get() == "foo" + assert q.get() == "bar" # self.assertEqual(q.alive, False) XXX queue don't know it's dead yet, but it is ... # Async get raises Empty (End is not returned) @@ -53,14 +53,14 @@ def test_input_runlevels(): assert q.alive # Before killing, let's slide some data in. - q.put('baz') + q.put("baz") # Now kill the queue... q.put(END) with pytest.raises(InactiveWritableError): - q.put('foo') + q.put("foo") # Still can get remaining data - assert q.get() == 'baz' + assert q.get() == "baz" with pytest.raises(InactiveReadableError): q.get() diff --git a/tests/structs/test_tokens.py b/tests/structs/test_tokens.py index 6bd9bee..0a222b3 100644 --- a/tests/structs/test_tokens.py +++ b/tests/structs/test_tokens.py @@ -2,5 +2,5 @@ from bonobo.structs.tokens import Token def test_token_repr(): - t = Token('Acme') - assert repr(t) == '' + t = Token("Acme") + assert repr(t) == "" diff --git a/tests/test_basicusage.py b/tests/test_basicusage.py index 7772af3..95e8628 100644 --- a/tests/test_basicusage.py +++ b/tests/test_basicusage.py @@ -11,7 +11,7 @@ def test_run_graph_noop(): graph = bonobo.Graph(bonobo.noop) assert len(graph) == 1 - with patch('bonobo._api._is_interactive_console', side_effect=lambda: False): + with patch("bonobo._api._is_interactive_console", side_effect=lambda: False): result = bonobo.run(graph) assert isinstance(result, GraphExecutionContext) diff --git a/tests/test_execution.py b/tests/test_execution.py index 3f51c45..8b22b3d 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -10,7 +10,7 @@ def generate_integers(): def square(i): - return i**2 + return i ** 2 def results(f, context): diff --git a/tests/test_publicapi.py b/tests/test_publicapi.py index 6b554e1..e2f3556 100644 --- a/tests/test_publicapi.py +++ b/tests/test_publicapi.py @@ -2,12 +2,12 @@ import inspect def test_wildcard_import(): - bonobo = __import__('bonobo') + bonobo = __import__("bonobo") assert bonobo.__version__ for name in dir(bonobo): # ignore attributes starting by underscores - if name.startswith('_'): + if name.startswith("_"): continue attr = getattr(bonobo, name) if inspect.ismodule(attr): diff --git a/tests/test_registry.py b/tests/test_registry.py index dfff5eb..9a0d53c 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,18 +1,18 @@ -from bonobo.nodes import CsvReader, JsonReader, CsvWriter, JsonWriter from bonobo import create_reader, create_writer +from bonobo.nodes import CsvReader, CsvWriter, JsonReader, JsonWriter def test_create_reader(): - t = create_reader('foo.csv') + t = create_reader("foo.csv") assert isinstance(t, CsvReader) - t = create_reader('foo.txt', format='json') + t = create_reader("foo.txt", format="json") assert isinstance(t, JsonReader) def test_create_writer(): - t = create_writer('foo.csv') + t = create_writer("foo.csv") assert isinstance(t, CsvWriter) - t = create_writer('foo.txt', format='json') + t = create_writer("foo.txt", format="json") assert isinstance(t, JsonWriter) diff --git a/tests/test_settings.py b/tests/test_settings.py index 5771dc4..4e49e57 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -6,37 +6,37 @@ import pytest from bonobo import settings -TEST_SETTING = 'TEST_SETTING' +TEST_SETTING = "TEST_SETTING" def test_to_bool(): - assert not settings.to_bool('') - assert not settings.to_bool('FALSE') - assert not settings.to_bool('NO') - assert not settings.to_bool('0') + assert not settings.to_bool("") + assert not settings.to_bool("FALSE") + assert not settings.to_bool("NO") + assert not settings.to_bool("0") - assert settings.to_bool('yup') - assert settings.to_bool('True') - assert settings.to_bool('yes') - assert settings.to_bool('1') + assert settings.to_bool("yup") + assert settings.to_bool("True") + assert settings.to_bool("yes") + assert settings.to_bool("1") def test_setting(): s = settings.Setting(TEST_SETTING) assert s.get() is None - with patch.dict(environ, {TEST_SETTING: 'hello'}): + with patch.dict(environ, {TEST_SETTING: "hello"}): assert s.get() is None s.clear() - assert s.get() == 'hello' + assert s.get() == "hello" - s = settings.Setting(TEST_SETTING, default='nope') - assert s.get() is 'nope' + s = settings.Setting(TEST_SETTING, default="nope") + assert s.get() is "nope" - with patch.dict(environ, {TEST_SETTING: 'hello'}): - assert s.get() == 'nope' + with patch.dict(environ, {TEST_SETTING: "hello"}): + assert s.get() == "nope" s.clear() - assert s.get() == 'hello' + assert s.get() == "hello" def test_default_settings(): @@ -45,18 +45,18 @@ def test_default_settings(): assert settings.DEBUG.get() is False assert settings.PROFILE.get() is False assert settings.QUIET.get() is False - assert settings.LOGGING_LEVEL.get() == logging._checkLevel('INFO') + assert settings.LOGGING_LEVEL.get() == logging._checkLevel("INFO") - with patch.dict(environ, {'DEBUG': 't'}): + with patch.dict(environ, {"DEBUG": "t"}): settings.clear_all() - assert settings.LOGGING_LEVEL.get() == logging._checkLevel('DEBUG') + assert settings.LOGGING_LEVEL.get() == logging._checkLevel("DEBUG") settings.clear_all() def test_check(): settings.check() - with patch.dict(environ, {'DEBUG': 't', 'PROFILE': 't', 'QUIET': 't'}): + with patch.dict(environ, {"DEBUG": "t", "PROFILE": "t", "QUIET": "t"}): settings.clear_all() with pytest.raises(RuntimeError): settings.check() diff --git a/tests/util/requireable/dummy.py b/tests/util/requireable/dummy.py index 1ce8ef1..05c8891 100644 --- a/tests/util/requireable/dummy.py +++ b/tests/util/requireable/dummy.py @@ -1 +1 @@ -foo = 'bar' +foo = "bar" diff --git a/tests/util/test_bags.py b/tests/util/test_bags.py index 469fbb6..80d93c8 100644 --- a/tests/util/test_bags.py +++ b/tests/util/test_bags.py @@ -16,49 +16,49 @@ from bonobo.util.bags import BagType ### Named Tuples ################################################################################ -TBag = BagType('TBag', ('x', 'y', 'z')) # type used for pickle tests +TBag = BagType("TBag", ("x", "y", "z")) # type used for pickle tests class TestBagType(unittest.TestCase): - def _create(self, *fields, typename='abc'): + def _create(self, *fields, typename="abc"): bt = BagType(typename, fields) assert bt._fields == fields assert len(bt._fields) == len(bt._attrs) return bt def test_factory(self): - Point = BagType('Point', ('x', 'y')) - self.assertEqual(Point.__name__, 'Point') + Point = BagType("Point", ("x", "y")) + self.assertEqual(Point.__name__, "Point") self.assertEqual(Point.__slots__, ()) self.assertEqual(Point.__module__, __name__) self.assertEqual(Point.__getitem__, tuple.__getitem__) - assert Point._fields == ('x', 'y') - assert Point._attrs == ('x', 'y') + assert Point._fields == ("x", "y") + assert Point._attrs == ("x", "y") - self.assertRaises(ValueError, BagType, 'abc%', ('efg', 'ghi')) # type has non-alpha char - self.assertRaises(ValueError, BagType, 'class', ('efg', 'ghi')) # type has keyword - self.assertRaises(ValueError, BagType, '9abc', ('efg', 'ghi')) # type starts with digit + self.assertRaises(ValueError, BagType, "abc%", ("efg", "ghi")) # type has non-alpha char + self.assertRaises(ValueError, BagType, "class", ("efg", "ghi")) # type has keyword + self.assertRaises(ValueError, BagType, "9abc", ("efg", "ghi")) # type starts with digit - assert self._create('efg', 'g%hi')._attrs == ('efg', 'g_hi') - assert self._create('abc', 'class')._attrs == ('abc', '_class') - assert self._create('8efg', '9ghi')._attrs == ('_8efg', '_9ghi') - assert self._create('_efg', 'ghi')._attrs == ('_efg', 'ghi') + assert self._create("efg", "g%hi")._attrs == ("efg", "g_hi") + assert self._create("abc", "class")._attrs == ("abc", "_class") + assert self._create("8efg", "9ghi")._attrs == ("_8efg", "_9ghi") + assert self._create("_efg", "ghi")._attrs == ("_efg", "ghi") - self.assertRaises(ValueError, BagType, 'abc', ('efg', 'efg', 'ghi')) # duplicate field + self.assertRaises(ValueError, BagType, "abc", ("efg", "efg", "ghi")) # duplicate field - self._create('x1', 'y2', typename='Point0') # Verify that numbers are allowed in names - self._create('a', 'b', 'c', typename='_') # Test leading underscores in a typename + self._create("x1", "y2", typename="Point0") # Verify that numbers are allowed in names + self._create("a", "b", "c", typename="_") # Test leading underscores in a typename - bt = self._create('a!', 'a?') - assert bt._attrs == ('a0', 'a1') - x = bt('foo', 'bar') - assert x.get('a!') == 'foo' - assert x.a0 == 'foo' - assert x.get('a?') == 'bar' - assert x.a1 == 'bar' + bt = self._create("a!", "a?") + assert bt._attrs == ("a0", "a1") + x = bt("foo", "bar") + assert x.get("a!") == "foo" + assert x.a0 == "foo" + assert x.get("a?") == "bar" + assert x.a1 == "bar" # check unicode output - bt = self._create('the', 'quick', 'brown', 'fox') + bt = self._create("the", "quick", "brown", "fox") assert "u'" not in repr(bt._fields) self.assertRaises(TypeError, Point._make, [11]) # catch too few args @@ -66,33 +66,33 @@ class TestBagType(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_factory_doc_attr(self): - Point = BagType('Point', ('x', 'y')) - self.assertEqual(Point.__doc__, 'Point(x, y)') + Point = BagType("Point", ("x", "y")) + self.assertEqual(Point.__doc__, "Point(x, y)") @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_doc_writable(self): - Point = BagType('Point', ('x', 'y')) + Point = BagType("Point", ("x", "y")) self.assertEqual(Point.x.__doc__, "Alias for 'x'") - Point.x.__doc__ = 'docstring for Point.x' - self.assertEqual(Point.x.__doc__, 'docstring for Point.x') + Point.x.__doc__ = "docstring for Point.x" + self.assertEqual(Point.x.__doc__, "docstring for Point.x") def test_name_fixer(self): for spec, renamed in [ - [('efg', 'g%hi'), ('efg', 'g_hi')], # field with non-alpha char - [('abc', 'class'), ('abc', '_class')], # field has keyword - [('8efg', '9ghi'), ('_8efg', '_9ghi')], # field starts with digit - [('abc', '_efg'), ('abc', '_efg')], # field with leading underscore - [('abc', '', 'x'), ('abc', '_0', 'x')], # fieldname is a space - [('&', '¨', '*'), ('_0', '_1', '_2')], # Duplicate attrs, in theory + [("efg", "g%hi"), ("efg", "g_hi")], # field with non-alpha char + [("abc", "class"), ("abc", "_class")], # field has keyword + [("8efg", "9ghi"), ("_8efg", "_9ghi")], # field starts with digit + [("abc", "_efg"), ("abc", "_efg")], # field with leading underscore + [("abc", "", "x"), ("abc", "_0", "x")], # fieldname is a space + [("&", "¨", "*"), ("_0", "_1", "_2")], # Duplicate attrs, in theory ]: assert self._create(*spec)._attrs == renamed def test_module_parameter(self): - NT = BagType('NT', ['x', 'y'], module=collections) + NT = BagType("NT", ["x", "y"], module=collections) self.assertEqual(NT.__module__, collections) def test_instance(self): - Point = self._create('x', 'y', typename='Point') + Point = self._create("x", "y", typename="Point") p = Point(11, 22) self.assertEqual(p, Point(x=11, y=22)) self.assertEqual(p, Point(11, y=22)) @@ -101,12 +101,12 @@ class TestBagType(unittest.TestCase): self.assertEqual(p, Point(**dict(x=11, y=22))) self.assertRaises(TypeError, Point, 1) # too few args self.assertRaises(TypeError, Point, 1, 2, 3) # too many args - self.assertRaises(TypeError, eval, 'Point(XXX=1, y=2)', locals()) # wrong keyword argument - self.assertRaises(TypeError, eval, 'Point(x=1)', locals()) # missing keyword argument - self.assertEqual(repr(p), 'Point(x=11, y=22)') - self.assertNotIn('__weakref__', dir(p)) + self.assertRaises(TypeError, eval, "Point(XXX=1, y=2)", locals()) # wrong keyword argument + self.assertRaises(TypeError, eval, "Point(x=1)", locals()) # missing keyword argument + self.assertEqual(repr(p), "Point(x=11, y=22)") + self.assertNotIn("__weakref__", dir(p)) self.assertEqual(p, Point._make([11, 22])) # test _make classmethod - self.assertEqual(p._fields, ('x', 'y')) # test _fields attribute + self.assertEqual(p._fields, ("x", "y")) # test _fields attribute self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method @@ -115,13 +115,13 @@ class TestBagType(unittest.TestCase): except ValueError: pass else: - self._fail('Did not detect an incorrect fieldname') + self._fail("Did not detect an incorrect fieldname") p = Point(x=11, y=22) - self.assertEqual(repr(p), 'Point(x=11, y=22)') + self.assertEqual(repr(p), "Point(x=11, y=22)") def test_tupleness(self): - Point = BagType('Point', ('x', 'y')) + Point = BagType("Point", ("x", "y")) p = Point(11, 22) self.assertIsInstance(p, tuple) @@ -137,29 +137,29 @@ class TestBagType(unittest.TestCase): self.assertEqual(p.x, x) self.assertEqual(p.y, y) - self.assertRaises(AttributeError, eval, 'p.z', locals()) + self.assertRaises(AttributeError, eval, "p.z", locals()) def test_odd_sizes(self): - Zero = BagType('Zero', ()) + Zero = BagType("Zero", ()) self.assertEqual(Zero(), ()) self.assertEqual(Zero._make([]), ()) - self.assertEqual(repr(Zero()), 'Zero()') + self.assertEqual(repr(Zero()), "Zero()") self.assertEqual(Zero()._asdict(), {}) self.assertEqual(Zero()._fields, ()) - Dot = BagType('Dot', ('d', )) - self.assertEqual(Dot(1), (1, )) - self.assertEqual(Dot._make([1]), (1, )) + Dot = BagType("Dot", ("d",)) + self.assertEqual(Dot(1), (1,)) + self.assertEqual(Dot._make([1]), (1,)) self.assertEqual(Dot(1).d, 1) - self.assertEqual(repr(Dot(1)), 'Dot(d=1)') - self.assertEqual(Dot(1)._asdict(), {'d': 1}) - self.assertEqual(Dot(1)._replace(d=999), (999, )) - self.assertEqual(Dot(1)._fields, ('d', )) + self.assertEqual(repr(Dot(1)), "Dot(d=1)") + self.assertEqual(Dot(1)._asdict(), {"d": 1}) + self.assertEqual(Dot(1)._replace(d=999), (999,)) + self.assertEqual(Dot(1)._fields, ("d",)) n = 5000 if sys.version_info >= (3, 7) else 254 - names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) + names = list(set("".join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) n = len(names) - Big = BagType('Big', names) + Big = BagType("Big", names) b = Big(*range(n)) self.assertEqual(b, tuple(range(n))) self.assertEqual(Big._make(range(n)), tuple(range(n))) @@ -178,14 +178,14 @@ class TestBagType(unittest.TestCase): def test_pickle(self): p = TBag(x=10, y=20, z=30) - for module in (pickle, ): - loads = getattr(module, 'loads') - dumps = getattr(module, 'dumps') + for module in (pickle,): + loads = getattr(module, "loads") + dumps = getattr(module, "dumps") for protocol in range(-1, module.HIGHEST_PROTOCOL + 1): q = loads(dumps(p, protocol)) self.assertEqual(p, q) self.assertEqual(p._fields, q._fields) - self.assertNotIn(b'OrderedDict', dumps(p, protocol)) + self.assertNotIn(b"OrderedDict", dumps(p, protocol)) def test_copy(self): p = TBag(x=10, y=20, z=30) @@ -197,7 +197,7 @@ class TestBagType(unittest.TestCase): def test_name_conflicts(self): # Some names like "self", "cls", "tuple", "itemgetter", and "property" # failed when used as field names. Test to make sure these now work. - T = BagType('T', ('itemgetter', 'property', 'self', 'cls', 'tuple')) + T = BagType("T", ("itemgetter", "property", "self", "cls", "tuple")) t = T(1, 2, 3, 4, 5) self.assertEqual(t, (1, 2, 3, 4, 5)) newt = t._replace(itemgetter=10, property=20, self=30, cls=40, tuple=50) @@ -206,28 +206,194 @@ class TestBagType(unittest.TestCase): # Broader test of all interesting names taken from the code, old # template, and an example words = { - 'Alias', 'At', 'AttributeError', 'Build', 'Bypass', 'Create', 'Encountered', 'Expected', 'Field', 'For', - 'Got', 'Helper', 'IronPython', 'Jython', 'KeyError', 'Make', 'Modify', 'Note', 'OrderedDict', 'Point', - 'Return', 'Returns', 'Type', 'TypeError', 'Used', 'Validate', 'ValueError', 'Variables', 'a', 'accessible', - 'add', 'added', 'all', 'also', 'an', 'arg_list', 'args', 'arguments', 'automatically', 'be', 'build', - 'builtins', 'but', 'by', 'cannot', 'class_namespace', 'classmethod', 'cls', 'collections', 'convert', - 'copy', 'created', 'creation', 'd', 'debugging', 'defined', 'dict', 'dictionary', 'doc', 'docstring', - 'docstrings', 'duplicate', 'effect', 'either', 'enumerate', 'environments', 'error', 'example', 'exec', 'f', - 'f_globals', 'field', 'field_names', 'fields', 'formatted', 'frame', 'function', 'functions', 'generate', - 'getter', 'got', 'greater', 'has', 'help', 'identifiers', 'indexable', 'instance', 'instantiate', - 'interning', 'introspection', 'isidentifier', 'isinstance', 'itemgetter', 'iterable', 'join', 'keyword', - 'keywords', 'kwds', 'len', 'like', 'list', 'map', 'maps', 'message', 'metadata', 'method', 'methods', - 'module', 'module_name', 'must', 'name', 'named', 'namedtuple', 'namedtuple_', 'names', 'namespace', - 'needs', 'new', 'nicely', 'num_fields', 'number', 'object', 'of', 'operator', 'option', 'p', 'particular', - 'pickle', 'pickling', 'plain', 'pop', 'positional', 'property', 'r', 'regular', 'rename', 'replace', - 'replacing', 'repr', 'repr_fmt', 'representation', 'result', 'reuse_itemgetter', 's', 'seen', 'sequence', - 'set', 'side', 'specified', 'split', 'start', 'startswith', 'step', 'str', 'string', 'strings', 'subclass', - 'sys', 'targets', 'than', 'the', 'their', 'this', 'to', 'tuple_new', 'type', 'typename', 'underscore', - 'unexpected', 'unpack', 'up', 'use', 'used', 'user', 'valid', 'values', 'variable', 'verbose', 'where', - 'which', 'work', 'x', 'y', 'z', 'zip' + "Alias", + "At", + "AttributeError", + "Build", + "Bypass", + "Create", + "Encountered", + "Expected", + "Field", + "For", + "Got", + "Helper", + "IronPython", + "Jython", + "KeyError", + "Make", + "Modify", + "Note", + "OrderedDict", + "Point", + "Return", + "Returns", + "Type", + "TypeError", + "Used", + "Validate", + "ValueError", + "Variables", + "a", + "accessible", + "add", + "added", + "all", + "also", + "an", + "arg_list", + "args", + "arguments", + "automatically", + "be", + "build", + "builtins", + "but", + "by", + "cannot", + "class_namespace", + "classmethod", + "cls", + "collections", + "convert", + "copy", + "created", + "creation", + "d", + "debugging", + "defined", + "dict", + "dictionary", + "doc", + "docstring", + "docstrings", + "duplicate", + "effect", + "either", + "enumerate", + "environments", + "error", + "example", + "exec", + "f", + "f_globals", + "field", + "field_names", + "fields", + "formatted", + "frame", + "function", + "functions", + "generate", + "getter", + "got", + "greater", + "has", + "help", + "identifiers", + "indexable", + "instance", + "instantiate", + "interning", + "introspection", + "isidentifier", + "isinstance", + "itemgetter", + "iterable", + "join", + "keyword", + "keywords", + "kwds", + "len", + "like", + "list", + "map", + "maps", + "message", + "metadata", + "method", + "methods", + "module", + "module_name", + "must", + "name", + "named", + "namedtuple", + "namedtuple_", + "names", + "namespace", + "needs", + "new", + "nicely", + "num_fields", + "number", + "object", + "of", + "operator", + "option", + "p", + "particular", + "pickle", + "pickling", + "plain", + "pop", + "positional", + "property", + "r", + "regular", + "rename", + "replace", + "replacing", + "repr", + "repr_fmt", + "representation", + "result", + "reuse_itemgetter", + "s", + "seen", + "sequence", + "set", + "side", + "specified", + "split", + "start", + "startswith", + "step", + "str", + "string", + "strings", + "subclass", + "sys", + "targets", + "than", + "the", + "their", + "this", + "to", + "tuple_new", + "type", + "typename", + "underscore", + "unexpected", + "unpack", + "up", + "use", + "used", + "user", + "valid", + "values", + "variable", + "verbose", + "where", + "which", + "work", + "x", + "y", + "z", + "zip", } sorted_words = tuple(sorted(words)) - T = BagType('T', sorted_words) + T = BagType("T", sorted_words) # test __new__ values = tuple(range(len(words))) t = T(*values) @@ -252,27 +418,39 @@ class TestBagType(unittest.TestCase): self.assertEqual(t.__getnewargs__(), values) def test_repr(self): - A = BagType('A', ('x', )) - self.assertEqual(repr(A(1)), 'A(x=1)') + A = BagType("A", ("x",)) + self.assertEqual(repr(A(1)), "A(x=1)") # repr should show the name of the subclass class B(A): pass - self.assertEqual(repr(B(1)), 'B(x=1)') + self.assertEqual(repr(B(1)), "B(x=1)") def test_namedtuple_subclass_issue_24931(self): - class Point(BagType('_Point', ['x', 'y'])): + class Point(BagType("_Point", ["x", "y"])): pass a = Point(3, 4) - self.assertEqual(a._asdict(), OrderedDict([('x', 3), ('y', 4)])) + self.assertEqual(a._asdict(), OrderedDict([("x", 3), ("y", 4)])) a.w = 5 - self.assertEqual(a.__dict__, {'w': 5}) + self.assertEqual(a.__dict__, {"w": 5}) def test_annoying_attribute_names(self): self._create( - '__slots__', '__getattr__', '_attrs', '_fields', '__new__', '__getnewargs__', '__repr__', '_make', 'get', - '_replace', '_asdict', '_cls', 'self', 'tuple' + "__slots__", + "__getattr__", + "_attrs", + "_fields", + "__new__", + "__getnewargs__", + "__repr__", + "_make", + "get", + "_replace", + "_asdict", + "_cls", + "self", + "tuple", ) diff --git a/tests/util/test_collections.py b/tests/util/test_collections.py index bdc6ec1..bd43ac2 100644 --- a/tests/util/test_collections.py +++ b/tests/util/test_collections.py @@ -1,7 +1,7 @@ import pytest -from bonobo.util import sortedlist, ensure_tuple -from bonobo.util.collections import tuplize, cast +from bonobo.util import ensure_tuple, sortedlist +from bonobo.util.collections import cast, tuplize def test_sortedlist(): @@ -14,20 +14,20 @@ def test_sortedlist(): def test_ensure_tuple(): - assert ensure_tuple('a') == ('a', ) - assert ensure_tuple(('a', )) == ('a', ) + assert ensure_tuple("a") == ("a",) + assert ensure_tuple(("a",)) == ("a",) assert ensure_tuple(()) is () -@pytest.mark.parametrize('tuplize', [tuplize, cast(tuple)]) +@pytest.mark.parametrize("tuplize", [tuplize, cast(tuple)]) def test_tuplize(tuplize): tuplized_lambda = tuplize(lambda: [1, 2, 3]) assert tuplized_lambda() == (1, 2, 3) @tuplize def some_generator(): - yield 'c' - yield 'b' - yield 'a' + yield "c" + yield "b" + yield "a" - assert some_generator() == ('c', 'b', 'a') + assert some_generator() == ("c", "b", "a") diff --git a/tests/util/test_compat.py b/tests/util/test_compat.py index 7be5dc3..647f765 100644 --- a/tests/util/test_compat.py +++ b/tests/util/test_compat.py @@ -17,7 +17,7 @@ def test_deprecated_alias(): def foo(): pass - foo = deprecated_alias('bar', foo) + foo = deprecated_alias("bar", foo) with pytest.warns(DeprecationWarning): foo() diff --git a/tests/util/test_objects.py b/tests/util/test_objects.py index 9a3696e..378bb85 100644 --- a/tests/util/test_objects.py +++ b/tests/util/test_objects.py @@ -2,7 +2,7 @@ import operator import pytest -from bonobo.util.objects import Wrapper, get_name, ValueHolder, get_attribute_or_create +from bonobo.util.objects import ValueHolder, Wrapper, get_attribute_or_create, get_name from bonobo.util.testing import optional_contextmanager @@ -11,31 +11,31 @@ class foo: class bar: - __name__ = 'baz' + __name__ = "baz" def test_get_name(): - assert get_name(42) == 'int' - assert get_name('eat at joe.') == 'str' - assert get_name(str) == 'str' - assert get_name(object) == 'object' - assert get_name(get_name) == 'get_name' - assert get_name(foo) == 'foo' - assert get_name(foo()) == 'foo' - assert get_name(bar) == 'bar' - assert get_name(bar()) == 'baz' + assert get_name(42) == "int" + assert get_name("eat at joe.") == "str" + assert get_name(str) == "str" + assert get_name(object) == "object" + assert get_name(get_name) == "get_name" + assert get_name(foo) == "foo" + assert get_name(foo()) == "foo" + assert get_name(bar) == "bar" + assert get_name(bar()) == "baz" def test_wrapper_name(): - assert get_name(Wrapper(42)) == 'int' - assert get_name(Wrapper('eat at joe.')) == 'str' - assert get_name(Wrapper(str)) == 'str' - assert get_name(Wrapper(object)) == 'object' - assert get_name(Wrapper(foo)) == 'foo' - assert get_name(Wrapper(foo())) == 'foo' - assert get_name(Wrapper(bar)) == 'bar' - assert get_name(Wrapper(bar())) == 'baz' - assert get_name(Wrapper(get_name)) == 'get_name' + assert get_name(Wrapper(42)) == "int" + assert get_name(Wrapper("eat at joe.")) == "str" + assert get_name(Wrapper(str)) == "str" + assert get_name(Wrapper(object)) == "object" + assert get_name(Wrapper(foo)) == "foo" + assert get_name(Wrapper(foo())) == "foo" + assert get_name(Wrapper(bar)) == "bar" + assert get_name(Wrapper(bar())) == "baz" + assert get_name(Wrapper(get_name)) == "get_name" def test_valueholder(): @@ -65,10 +65,7 @@ def test_valueholder_notequal(): assert not (x != 42) -@pytest.mark.parametrize('rlo,rhi', [ - (1, 2), - ('a', 'b'), -]) +@pytest.mark.parametrize("rlo,rhi", [(1, 2), ("a", "b")]) def test_valueholder_ordering(rlo, rhi): vlo, vhi = ValueHolder(rlo), ValueHolder(rhi) @@ -97,15 +94,15 @@ def test_valueholders_containers(): assert 5 in x assert 42 not in x - y = ValueHolder({'foo': 'bar', 'corp': 'acme'}) + y = ValueHolder({"foo": "bar", "corp": "acme"}) - assert 'foo' in y - assert y['foo'] == 'bar' + assert "foo" in y + assert y["foo"] == "bar" with pytest.raises(KeyError): - y['no'] - y['no'] = 'oh, wait' - assert 'no' in y - assert 'oh, wait' == y['no'] + y["no"] + y["no"] = "oh, wait" + assert "no" in y + assert "oh, wait" == y["no"] def test_get_attribute_or_create(): @@ -117,27 +114,39 @@ def test_get_attribute_or_create(): with pytest.raises(AttributeError): x.foo - foo = get_attribute_or_create(x, 'foo', 'bar') - assert foo == 'bar' - assert x.foo == 'bar' + foo = get_attribute_or_create(x, "foo", "bar") + assert foo == "bar" + assert x.foo == "bar" - foo = get_attribute_or_create(x, 'foo', 'baz') - assert foo == 'bar' - assert x.foo == 'bar' + foo = get_attribute_or_create(x, "foo", "baz") + assert foo == "bar" + assert x.foo == "bar" unsupported_operations = { int: {operator.matmul}, str: { - operator.sub, operator.mul, operator.matmul, operator.floordiv, operator.truediv, operator.mod, divmod, - operator.pow, operator.lshift, operator.rshift, operator.and_, operator.xor, operator.or_ + operator.sub, + operator.mul, + operator.matmul, + operator.floordiv, + operator.truediv, + operator.mod, + divmod, + operator.pow, + operator.lshift, + operator.rshift, + operator.and_, + operator.xor, + operator.or_, }, } -@pytest.mark.parametrize('x,y', [(5, 3), (0, 10), (0, 0), (1, 1), ('foo', 'bar'), ('', 'baz!')]) +@pytest.mark.parametrize("x,y", [(5, 3), (0, 10), (0, 0), (1, 1), ("foo", "bar"), ("", "baz!")]) @pytest.mark.parametrize( - 'operation,inplace_operation', [ + "operation,inplace_operation", + [ (operator.add, operator.iadd), (operator.sub, operator.isub), (operator.mul, operator.imul), @@ -152,14 +161,14 @@ unsupported_operations = { (operator.and_, operator.iand), (operator.xor, operator.ixor), (operator.or_, operator.ior), - ] + ], ) def test_valueholder_integer_operations(x, y, operation, inplace_operation): v = ValueHolder(x) is_supported = operation not in unsupported_operations.get(type(x), set()) - isdiv = ('div' in operation.__name__) or ('mod' in operation.__name__) + isdiv = ("div" in operation.__name__) or ("mod" in operation.__name__) # forward... with optional_contextmanager(pytest.raises(TypeError), ignore=is_supported): diff --git a/tests/util/test_resolvers.py b/tests/util/test_resolvers.py index 0de3003..4f584a7 100644 --- a/tests/util/test_resolvers.py +++ b/tests/util/test_resolvers.py @@ -3,16 +3,16 @@ from bonobo.util.resolvers import _parse_option, _resolve_options, _resolve_tran def test_parse_option(): - assert _parse_option('foo=bar') == ('foo', 'bar') - assert _parse_option('foo="bar"') == ('foo', 'bar') - assert _parse_option('sep=";"') == ('sep', ';') - assert _parse_option('foo') == ('foo', True) + assert _parse_option("foo=bar") == ("foo", "bar") + assert _parse_option('foo="bar"') == ("foo", "bar") + assert _parse_option('sep=";"') == ("sep", ";") + assert _parse_option("foo") == ("foo", True) def test_resolve_options(): - assert _resolve_options(('foo=bar', 'bar="baz"')) == {'foo': 'bar', 'bar': 'baz'} + assert _resolve_options(("foo=bar", 'bar="baz"')) == {"foo": "bar", "bar": "baz"} assert _resolve_options() == {} def test_resolve_transformations(): - assert _resolve_transformations(('PrettyPrinter', )) == (bonobo.PrettyPrinter, ) + assert _resolve_transformations(("PrettyPrinter",)) == (bonobo.PrettyPrinter,) diff --git a/tests/util/test_statistics.py b/tests/util/test_statistics.py index 9eae0c0..a8d9fa6 100644 --- a/tests/util/test_statistics.py +++ b/tests/util/test_statistics.py @@ -3,12 +3,9 @@ from bonobo.util.statistics import WithStatistics class MyThingWithStats(WithStatistics): def get_statistics(self, *args, **kwargs): - return ( - ('foo', 42), - ('bar', 69), - ) + return (("foo", 42), ("bar", 69)) def test_with_statistics(): o = MyThingWithStats() - assert o.get_statistics_as_string() == 'foo=42 bar=69' + assert o.get_statistics_as_string() == "foo=42 bar=69"