From cdb602c825d78c443a56c5803da849393f494922 Mon Sep 17 00:00:00 2001 From: Obiamaka Agbaneje Date: Sat, 28 Jul 2018 17:25:39 +0100 Subject: [PATCH 01/35] Correct a few typos --- docs/tutorial/3-files.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorial/3-files.rst b/docs/tutorial/3-files.rst index 93cbd96..f98d12e 100644 --- a/docs/tutorial/3-files.rst +++ b/docs/tutorial/3-files.rst @@ -44,7 +44,7 @@ Now, we need to write a `writer` transformation, and apply this context processo f.write(repr(row) + "\n") The `f` parameter will contain the value yielded by the context processors, in order of appearance. You can chain -multiple context processors. To find about how to implement this, check the |bonobo| guides in the documentation. +multiple context processors. To find out about how to implement this, check the |bonobo| guides in the documentation. Please note that the :func:`bonobo.config.use_context_processor` decorator will modify the function in place, but won't modify its behaviour. If you want to call it out of the |bonobo| job context, it's your responsibility to provide @@ -144,7 +144,7 @@ Reading from files is done using the same logic as writing, except that you'll p def get_graph(**options): graph = bonobo.Graph() graph.add_chain( - bonobo.CsvReader('output.csv'), + bonobo.CsvReader('input.csv'), ... ) return graph From 499ebba4219e412f38eda66878543f2ac9d05d30 Mon Sep 17 00:00:00 2001 From: Obiamaka Agbaneje Date: Sat, 28 Jul 2018 17:26:20 +0100 Subject: [PATCH 02/35] Rephrase a few sentences --- docs/tutorial/4-services.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/tutorial/4-services.rst b/docs/tutorial/4-services.rst index 51b8683..9621df8 100644 --- a/docs/tutorial/4-services.rst +++ b/docs/tutorial/4-services.rst @@ -2,9 +2,8 @@ Part 4: Services ================ All external dependencies (like filesystems, network clients, database connections, etc.) should be provided to -transformations as a service. It allows great flexibility, including the ability to test your transformations isolated -from the external world, and being friendly to the infrastructure people (and if you're one of them, it's also nice to -treat yourself well). +transformations as a service. This will allow for great flexibility, including the ability to test your transformations isolated +from the external world and easily switch to production (being user-friendly for people in system administration). In the last section, we used the `fs` service to access filesystems, we'll go even further by switching our `requests` call to use the `http` service, so we can switch the `requests` session at runtime. We'll use it to add an http cache, @@ -24,7 +23,7 @@ Overriding services ::::::::::::::::::: You can override the default services, or define your own services, by providing a dictionary to the `services=` -argument of :obj:`bonobo.run`: +argument of :obj:`bonobo.run`. First, let's rewrite get_services: .. code-block:: python @@ -50,8 +49,8 @@ Let's replace the :obj:`requests.get` call we used in the first steps to use the def extract_fablabs(http): yield from http.get(FABLABS_API_URL).json().get('records') -Tadaa, done! You're not anymore tied to a specific implementation, but to whatever :obj:`requests` compatible object the -user want to provide. +Tadaa, done! You're no more tied to a specific implementation, but to whatever :obj:`requests` -compatible object the +user wants to provide. Adding cache :::::::::::: From b1c85a80b01d14b1ce64e373125bab034927ed4e Mon Sep 17 00:00:00 2001 From: Obiamaka Agbaneje Date: Sat, 28 Jul 2018 17:37:36 +0100 Subject: [PATCH 03/35] Add link to python generators page --- docs/tutorial/1-init.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/1-init.rst b/docs/tutorial/1-init.rst index f0f7fbf..1668417 100644 --- a/docs/tutorial/1-init.rst +++ b/docs/tutorial/1-init.rst @@ -112,7 +112,7 @@ Extract yield 'hello' yield 'world' -This is a first transformation, written as a python generator, that will send some strings, one after the other, to its +This is a first transformation, written as a `python generator `_, that will send some strings, one after the other, to its output. Transformations that take no input and yields a variable number of outputs are usually called **extractors**. You'll From e429bb24ce7d3e80a80dec3548ee7d70ccc50f95 Mon Sep 17 00:00:00 2001 From: Obiamaka Agbaneje Date: Sat, 28 Jul 2018 17:48:06 +0100 Subject: [PATCH 04/35] Rephrase some sentences --- docs/tutorial/5-packaging.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/tutorial/5-packaging.rst b/docs/tutorial/5-packaging.rst index d6556e1..6e1ba72 100644 --- a/docs/tutorial/5-packaging.rst +++ b/docs/tutorial/5-packaging.rst @@ -1,9 +1,7 @@ Part 5: Projects and Packaging ============================== -Until then, we worked with one file managing a job. - -Real life often involves more complicated setups, with relations and imports between different files. +Throughout this tutorial, we have been working with one file managing a job but real life often involves more complicated setups, with relations and imports between different files. Data processing is something a wide variety of tools may want to include, and thus |bonobo| does not enforce any kind of project structure, as the target structure will be dictated by the hosting project. For example, a `pipelines` @@ -17,7 +15,7 @@ Imports mechanism |bonobo| does not enforce anything on how the python import mechanism work. Especially, it won't add anything to your `sys.path`, unlike some popular projects, because we're not sure that's something you want. -If you want to use imports, you should move your script in a python package, and it's up to you to have it setup +If you want to use imports, you should move your script into a python package, and it's up to you to have it setup correctly. @@ -36,8 +34,8 @@ your jobs in it. For example, it can be `mypkg.pipelines`. Creating a brand new package :::::::::::::::::::::::::::: -Because you're maybe starting a project with the data-engineering part, then you may not have a python package yet. As -it can be a bit tedious to setup right, there is an helper, using `Medikit `_, that +Because you may be starting a project involving some data-engineering, you may not have a python package yet. As +it can be a bit tedious to setup right, there is a helper, using `Medikit `_, that you can use to create a brand new project: .. code-block:: shell-session @@ -72,7 +70,7 @@ created in this tutorial and extend it): * :doc:`/extension/jupyter` * :doc:`/extension/sqlalchemy` -Then, you can either to jump head-first into your code, or you can have a better grasp at all concepts by +Then, you can either jump head-first into your code, or you can have a better grasp at all concepts by :doc:`reading the full bonobo guide `. You should also `join the slack community `_ and ask all your questions there! No From 43ed9ad154a40ca8623018542a6c7d888ca7e20a Mon Sep 17 00:00:00 2001 From: Obiamaka Agbaneje Date: Sun, 29 Jul 2018 16:08:45 +0100 Subject: [PATCH 05/35] Delete old files --- bonobo/examples/datasets/coffeeshops.py | 5 ++ bonobo/examples/datasets/fablabs.py | 2 +- bonobo/examples/tutorials/__init__.py | 0 bonobo/examples/tutorials/_services.py | 5 -- bonobo/examples/tutorials/tut01e01.py | 23 --------- bonobo/examples/tutorials/tut01e02.py | 14 ------ bonobo/examples/tutorials/tut02e01_read.py | 14 ------ bonobo/examples/tutorials/tut02e02_write.py | 23 --------- .../examples/tutorials/tut02e03_writeasmap.py | 36 ------------- .../tutorials/tutorial_basics_firststeps.py | 25 ---------- .../tutorials/tutorial_basics_summary.py | 11 ---- docs/reference/examples.rst | 14 +++++- docs/reference/examples/tutorials.rst | 50 ------------------- 13 files changed, 19 insertions(+), 203 deletions(-) delete mode 100644 bonobo/examples/tutorials/__init__.py delete mode 100644 bonobo/examples/tutorials/_services.py delete mode 100644 bonobo/examples/tutorials/tut01e01.py delete mode 100644 bonobo/examples/tutorials/tut01e02.py delete mode 100644 bonobo/examples/tutorials/tut02e01_read.py delete mode 100644 bonobo/examples/tutorials/tut02e02_write.py delete mode 100644 bonobo/examples/tutorials/tut02e03_writeasmap.py delete mode 100644 bonobo/examples/tutorials/tutorial_basics_firststeps.py delete mode 100644 bonobo/examples/tutorials/tutorial_basics_summary.py delete mode 100644 docs/reference/examples/tutorials.rst diff --git a/bonobo/examples/datasets/coffeeshops.py b/bonobo/examples/datasets/coffeeshops.py index 93aa0d5..fdca3cd 100644 --- a/bonobo/examples/datasets/coffeeshops.py +++ b/bonobo/examples/datasets/coffeeshops.py @@ -5,6 +5,11 @@ from bonobo.examples.datasets.services import get_services def get_graph(graph=None, *, _limit=(), _print=()): + """ + Extracts a list of cafes with on euro in Paris, renames the name, address and zipcode fields, + reorders the fields and formats to json and csv files. + + """ graph = graph or bonobo.Graph() producer = graph.add_chain( diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index 9a02593..9d2f4ed 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -1,5 +1,5 @@ """ -Extracts a list of fablabs in the world, restrict to the ones in france, then format it both for a nice console output +Extracts a list of fablabs in the world, restricted to the ones in france, then format its both for a nice console output and a flat txt file. .. graphviz:: diff --git a/bonobo/examples/tutorials/__init__.py b/bonobo/examples/tutorials/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bonobo/examples/tutorials/_services.py b/bonobo/examples/tutorials/_services.py deleted file mode 100644 index 25d783d..0000000 --- a/bonobo/examples/tutorials/_services.py +++ /dev/null @@ -1,5 +0,0 @@ -from bonobo import open_examples_fs - - -def get_services(): - return {'fs': open_examples_fs('datasets')} diff --git a/bonobo/examples/tutorials/tut01e01.py b/bonobo/examples/tutorials/tut01e01.py deleted file mode 100644 index c524039..0000000 --- a/bonobo/examples/tutorials/tut01e01.py +++ /dev/null @@ -1,23 +0,0 @@ -import bonobo - - -def extract(): - yield 'foo' - yield 'bar' - yield 'baz' - - -def transform(x): - return x.upper() - - -def load(x): - print(x) - - -graph = bonobo.Graph(extract, transform, load) - -graph.__doc__ = 'hello' - -if __name__ == '__main__': - bonobo.run(graph) diff --git a/bonobo/examples/tutorials/tut01e02.py b/bonobo/examples/tutorials/tut01e02.py deleted file mode 100644 index 78b7f43..0000000 --- a/bonobo/examples/tutorials/tut01e02.py +++ /dev/null @@ -1,14 +0,0 @@ -import bonobo - -graph = bonobo.Graph( - [ - 'foo', - 'bar', - 'baz', - ], - str.upper, - print, -) - -if __name__ == '__main__': - bonobo.run(graph) diff --git a/bonobo/examples/tutorials/tut02e01_read.py b/bonobo/examples/tutorials/tut02e01_read.py deleted file mode 100644 index 362051a..0000000 --- a/bonobo/examples/tutorials/tut02e01_read.py +++ /dev/null @@ -1,14 +0,0 @@ -import bonobo - -graph = bonobo.Graph( - bonobo.FileReader('coffeeshops.txt'), - print, -) - - -def get_services(): - return {'fs': bonobo.open_examples_fs('datasets')} - - -if __name__ == '__main__': - bonobo.run(graph, services=get_services()) diff --git a/bonobo/examples/tutorials/tut02e02_write.py b/bonobo/examples/tutorials/tut02e02_write.py deleted file mode 100644 index a33a11b..0000000 --- a/bonobo/examples/tutorials/tut02e02_write.py +++ /dev/null @@ -1,23 +0,0 @@ -import bonobo - - -def split_one(line): - return dict(zip(("name", "address"), line.split(', ', 1))) - - -graph = bonobo.Graph( - bonobo.FileReader('coffeeshops.txt'), - split_one, - bonobo.JsonWriter('coffeeshops.json', fs='fs.output'), -) - - -def get_services(): - return { - 'fs': bonobo.open_examples_fs('datasets'), - 'fs.output': bonobo.open_fs(), - } - - -if __name__ == '__main__': - bonobo.run(graph, services=get_services()) diff --git a/bonobo/examples/tutorials/tut02e03_writeasmap.py b/bonobo/examples/tutorials/tut02e03_writeasmap.py deleted file mode 100644 index afc251e..0000000 --- a/bonobo/examples/tutorials/tut02e03_writeasmap.py +++ /dev/null @@ -1,36 +0,0 @@ -import json - -import bonobo - - -def split_one_to_map(line): - k, v = line.split(', ', 1) - return {k: v} - - -class MyJsonWriter(bonobo.JsonWriter): - prefix, suffix = '{', '}' - - def write(self, fs, file, lineno, **row): - return bonobo.FileWriter.write( - self, fs, file, lineno, - json.dumps(row)[1:-1] - ) - - -graph = bonobo.Graph( - bonobo.FileReader('coffeeshops.txt'), - split_one_to_map, - MyJsonWriter('coffeeshops.json', fs='fs.output'), -) - - -def get_services(): - return { - 'fs': bonobo.open_examples_fs('datasets'), - 'fs.output': bonobo.open_fs(), - } - - -if __name__ == '__main__': - bonobo.run(graph, services=get_services()) diff --git a/bonobo/examples/tutorials/tutorial_basics_firststeps.py b/bonobo/examples/tutorials/tutorial_basics_firststeps.py deleted file mode 100644 index d024287..0000000 --- a/bonobo/examples/tutorials/tutorial_basics_firststeps.py +++ /dev/null @@ -1,25 +0,0 @@ -import bonobo - - -def generate_data(): - yield 'foo' - yield 'bar' - yield 'baz' - - -def uppercase(x: str): - return x.upper() - - -def output(x: str): - print(x) - - -graph = bonobo.Graph( - generate_data, - uppercase, - output, -) - -if __name__ == '__main__': - bonobo.run(graph) diff --git a/bonobo/examples/tutorials/tutorial_basics_summary.py b/bonobo/examples/tutorials/tutorial_basics_summary.py deleted file mode 100644 index a75e0f5..0000000 --- a/bonobo/examples/tutorials/tutorial_basics_summary.py +++ /dev/null @@ -1,11 +0,0 @@ -import bonobo - -# Represent our data processor as a simple directed graph of callables. -graph = bonobo.Graph( - ['foo', 'bar', 'baz'], - str.upper, - print, -) - -if __name__ == '__main__': - bonobo.run(graph) diff --git a/docs/reference/examples.rst b/docs/reference/examples.rst index b36c414..4edc910 100644 --- a/docs/reference/examples.rst +++ b/docs/reference/examples.rst @@ -5,7 +5,19 @@ There are a few examples bundled with **bonobo**. You'll find them under the :mod:`bonobo.examples` package, and you can run them directly as modules: - $ bonobo run -m bonobo.examples...module +.. code-block:: shell-session + + $ bonobo run -m bonobo.examples.module + + +or + +.. code-block:: shell-session + + $ python -m bonobo.examples.module + + + .. toctree:: :maxdepth: 4 diff --git a/docs/reference/examples/tutorials.rst b/docs/reference/examples/tutorials.rst deleted file mode 100644 index dbee6fd..0000000 --- a/docs/reference/examples/tutorials.rst +++ /dev/null @@ -1,50 +0,0 @@ -Examples from the tutorial -========================== - -Examples from :doc:`/tutorial/tut01` -:::::::::::::::::::::::::::::::::::: - -Example 1 ---------- - -.. automodule:: bonobo.examples.tutorials.tut01e01 - :members: - :undoc-members: - :show-inheritance: - -Example 2 ---------- - -.. automodule:: bonobo.examples.tutorials.tut01e02 - :members: - :undoc-members: - :show-inheritance: - -Examples from :doc:`/tutorial/tut02` -:::::::::::::::::::::::::::::::::::: - -Example 1: Read ---------------- - -.. automodule:: bonobo.examples.tutorials.tut02e01_read - :members: - :undoc-members: - :show-inheritance: - -Example 2: Write ----------------- - -.. automodule:: bonobo.examples.tutorials.tut02e02_write - :members: - :undoc-members: - :show-inheritance: - -Example 3: Write as map ------------------------ - -.. automodule:: bonobo.examples.tutorials.tut02e03_writeasmap - :members: - :undoc-members: - :show-inheritance: - - From 71f275910bfa323986ade9d261291ee7f6644dc8 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Wed, 8 Aug 2018 15:13:30 +0200 Subject: [PATCH 06/35] Fix erroneous parameter name. --- bonobo/contrib/django/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py index ac5f387..828c3a3 100644 --- a/bonobo/contrib/django/commands.py +++ b/bonobo/contrib/django/commands.py @@ -74,6 +74,6 @@ class ETLCommand(BaseCommand): self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr, ending=CLEAR_EOL + '\n') self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x - self.run(*args, **kwargs) + self.run(*args, **options) self.stdout, self.stderr = _stdout_backup, _stderr_backup From ebba06822bdc31ebe4817691bf578a25fac4082b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:09:36 +0200 Subject: [PATCH 07/35] style: changing assert in favor of exception --- bonobo/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 704c9ef..e3e46b7 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -7,7 +7,8 @@ import sys -assert (sys.version_info >= (3, 5)), 'Python 3.5+ is required to use Bonobo.' +if sys.version_info < (3, 5): + raise RuntimeError('Python 3.5+ is required to use Bonobo.') from bonobo._api import * from bonobo._api import __all__ From d1c9beae976567181694f93c9601fca6b372d40a Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:34:37 +0200 Subject: [PATCH 08/35] style: switching from yapf to isort/black --- .editorconfig | 17 ++ Makefile | 26 ++- Projectfile | 26 ++- benchmarks/parameters.py | 5 +- bin/update_apidoc.py | 22 +-- bonobo/__init__.py | 4 +- bonobo/_api.py | 11 +- bonobo/commands/__init__.py | 1 + bonobo/commands/base.py | 1 + bonobo/commands/convert.py | 45 ++--- bonobo/commands/run.py | 2 + bonobo/commands/version.py | 3 +- bonobo/config/configurables.py | 23 ++- bonobo/config/functools.py | 1 - bonobo/config/options.py | 9 +- bonobo/config/processors.py | 20 +- bonobo/config/services.py | 3 + bonobo/contrib/django/__init__.py | 5 +- bonobo/contrib/django/commands.py | 11 +- bonobo/contrib/google/__init__.py | 4 +- bonobo/contrib/jupyter/__init__.py | 4 +- bonobo/contrib/opendatasoft/__init__.py | 10 +- bonobo/errors.py | 8 +- bonobo/examples/__init__.py | 19 +- bonobo/examples/clock.py | 8 +- bonobo/examples/datasets/__main__.py | 31 +-- bonobo/examples/datasets/coffeeshops.py | 39 +--- bonobo/examples/datasets/fablabs.py | 17 +- bonobo/examples/datasets/services.py | 4 +- bonobo/examples/files/_services.py | 5 +- bonobo/examples/files/csv_handlers.py | 23 +-- bonobo/examples/files/json_handlers.py | 34 +--- bonobo/examples/files/pickle_handlers.py | 15 +- bonobo/examples/files/text_handlers.py | 5 +- bonobo/execution/contexts/__init__.py | 6 +- bonobo/execution/contexts/graph.py | 3 +- bonobo/execution/contexts/node.py | 9 +- bonobo/execution/strategies/__init__.py | 4 +- bonobo/execution/strategies/base.py | 1 + bonobo/nodes/basics.py | 28 +-- bonobo/nodes/filter.py | 3 +- bonobo/nodes/io/base.py | 37 +++- bonobo/nodes/io/csv.py | 6 +- bonobo/nodes/io/file.py | 22 ++- bonobo/plugins/console.py | 56 +++--- bonobo/registry.py | 3 +- bonobo/settings.py | 8 +- bonobo/structs/__init__.py | 4 +- bonobo/structs/graphs.py | 6 +- bonobo/util/__init__.py | 2 +- bonobo/util/api.py | 12 +- bonobo/util/bags.py | 19 +- bonobo/util/collections.py | 2 +- bonobo/util/compat.py | 2 +- bonobo/util/environ.py | 2 + bonobo/util/inspect.py | 22 +-- bonobo/util/objects.py | 4 +- bonobo/util/term.py | 2 +- bonobo/util/testing.py | 24 +-- docs/_templates/alabaster/support.py | 8 +- docs/conf.py | 33 ++-- requirements-dev.txt | 29 +-- requirements-docker.txt | 22 +-- requirements-jupyter.txt | 17 +- requirements-sqlalchemy.txt | 20 +- requirements.txt | 18 +- setup.py | 60 ++++-- tests/commands/test_clibasics.py | 10 +- tests/commands/test_download.py | 7 +- tests/commands/test_run_environ.py | 21 +- tests/commands/test_version.py | 2 +- tests/config/test_methods.py | 25 +-- tests/config/test_methods_partial.py | 7 +- tests/config/test_processors.py | 2 +- tests/config/test_services.py | 46 +++-- tests/execution/contexts/test_node.py | 34 ++-- tests/ext/test_ods.py | 9 +- tests/features/test_inherit.py | 9 +- tests/features/test_not_modified.py | 5 +- tests/nodes/io/test_csv.py | 63 ++---- tests/nodes/io/test_file.py | 9 +- tests/nodes/io/test_json.py | 96 ++-------- tests/nodes/io/test_pickle.py | 5 +- tests/nodes/test_basics.py | 15 +- tests/plugins/test_console.py | 3 +- tests/structs/test_graphs.py | 18 +- tests/structs/test_inputs.py | 2 +- tests/test_execution.py | 2 +- tests/util/test_bags.py | 234 ++++++++++++++++++++--- tests/util/test_collections.py | 8 +- tests/util/test_objects.py | 27 ++- tests/util/test_resolvers.py | 2 +- tests/util/test_statistics.py | 5 +- 93 files changed, 805 insertions(+), 816 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..428700d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 + +[*.py] +indent = ' ' +indent_size = 4 +indent_style = space +line_length = 120 +multi_line_output = 5 + +[Makefile] +indent_style = tab + diff --git a/Makefile b/Makefile index 42c8835..5f078ed 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Generated by Medikit 0.6.1 on 2018-05-21. +# Generated by Medikit 0.6.3 on 2018-08-11. # All changes will be overriden. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate. @@ -26,12 +26,10 @@ SPHINX_BUILD ?= $(PYTHON_DIRNAME)/sphinx-build SPHINX_OPTIONS ?= SPHINX_SOURCEDIR ?= docs SPHINX_BUILDDIR ?= $(SPHINX_SOURCEDIR)/_build -YAPF ?= $(PYTHON) -m yapf -YAPF_OPTIONS ?= -rip SPHINX_AUTOBUILD ?= $(PYTHON_DIRNAME)/sphinx-autobuild MEDIKIT ?= $(PYTHON) -m medikit MEDIKIT_UPDATE_OPTIONS ?= -MEDIKIT_VERSION ?= 0.6.1 +MEDIKIT_VERSION ?= 0.6.3 .PHONY: $(SPHINX_SOURCEDIR) clean format help install install-dev install-docker install-jupyter install-sqlalchemy medikit quick test update update-requirements watch-$(SPHINX_SOURCEDIR) @@ -44,7 +42,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_INLINE) -r $(PYTHON_REQUIREMENTS_FILE) @mkdir -p .medikit; touch $@ endif @@ -62,7 +60,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DEV_INLINE) -r $(PYTHON_REQUIREMENTS_DEV_FILE) @mkdir -p .medikit; touch $@ endif @@ -79,7 +77,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_DOCKER_INLINE) -r $(PYTHON_REQUIREMENTS_DOCKER_FILE) @mkdir -p .medikit; touch $@ endif @@ -93,7 +91,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_JUPYTER_INLINE) -r $(PYTHON_REQUIREMENTS_JUPYTER_FILE) @mkdir -p .medikit; touch $@ endif @@ -107,7 +105,7 @@ else ifneq ($(QUICK),) @printf "Skipping \033[36m%s\033[0m because \033[36m$$QUICK\033[0m is not empty.\n" $(target) else @printf "Applying \033[36m%s\033[0m target...\n" $(target) - $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=10.0" wheel + $(PIP) install $(PIP_INSTALL_OPTIONS) -U "pip ~=18.0" wheel $(PIP) install $(PIP_INSTALL_OPTIONS) -U $(PYTHON_REQUIREMENTS_SQLALCHEMY_INLINE) -r $(PYTHON_REQUIREMENTS_SQLALCHEMY_FILE) @mkdir -p .medikit; touch $@ endif @@ -118,15 +116,15 @@ test: install-dev ## Runs the test suite. $(SPHINX_SOURCEDIR): install-dev ## $(SPHINX_BUILD) -b html -D latex_paper_size=a4 $(SPHINX_OPTIONS) $(SPHINX_SOURCEDIR) $(SPHINX_BUILDDIR)/html -format: install-dev ## Reformats the whole python codebase using yapf. - $(YAPF) $(YAPF_OPTIONS) . - $(YAPF) $(YAPF_OPTIONS) Projectfile - watch-$(SPHINX_SOURCEDIR): ## $(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d) +format: ## Reformats the whole codebase using our standards (requires black and isort). + black -l 120 --skip-string-normalization . + isort -rc -o mondrian -o whistle -y . + medikit: # Checks installed medikit version and updates it if it is outdated. - @$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=10.0" "medikit>=$(MEDIKIT_VERSION)" + @$(PYTHON) -c 'import medikit, pip, sys; from packaging.version import Version; sys.exit(0 if (Version(medikit.__version__) >= Version("$(MEDIKIT_VERSION)")) and (Version(pip.__version__) < Version("10")) else 1)' || $(PYTHON) -m pip install -U "pip ~=18.0" "medikit>=$(MEDIKIT_VERSION)" update: medikit ## Update project artifacts using medikit. $(MEDIKIT) update $(MEDIKIT_UPDATE_OPTIONS) diff --git a/Projectfile b/Projectfile index 938f3d9..d55fa3e 100644 --- a/Projectfile +++ b/Projectfile @@ -6,7 +6,6 @@ make = require('make') pytest = require('pytest') python = require('python') sphinx = require('sphinx') -yapf = require('yapf') python.setup( name='bonobo', @@ -72,14 +71,29 @@ python.add_requirements( @listen(make.on_generate) def on_make_generate(event): - event.makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild' - event.makefile.add_target( + makefile = event.makefile + + # Sphinx + makefile['SPHINX_AUTOBUILD'] = '$(PYTHON_DIRNAME)/sphinx-autobuild' + makefile.add_target( 'watch-$(SPHINX_SOURCEDIR)', - ''' - $(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d) - ''', + '$(SPHINX_AUTOBUILD) $(SPHINX_SOURCEDIR) $(shell mktemp -d)', phony=True ) + # Formating + makefile.add_target( + 'format', + ''' + black -l 120 --skip-string-normalization . + isort -rc -o mondrian -o whistle -y . + ''', + phony=True, + doc='Reformats the whole codebase using our standards (requires black and isort).' + ) + + + + # vim: ft=python: diff --git a/benchmarks/parameters.py b/benchmarks/parameters.py index f51e389..f8e63a1 100644 --- a/benchmarks/parameters.py +++ b/benchmarks/parameters.py @@ -47,10 +47,9 @@ if __name__ == '__main__': for i in 1, 2, 3: print( - 'j{}'.format(i), - timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i)) + 'j{}'.format(i), timeit.timeit("j{}({!r})".format(i, json_data), setup="from __main__ import j{}".format(i)) ) print( 'k{}'.format(i), - timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)) + timeit.timeit("k{}(**{!r})".format(i, json_data), setup="from __main__ import k{}".format(i)), ) diff --git a/bin/update_apidoc.py b/bin/update_apidoc.py index a03dc63..4ed763f 100644 --- a/bin/update_apidoc.py +++ b/bin/update_apidoc.py @@ -1,6 +1,6 @@ import os -from jinja2 import Environment, DictLoader +from jinja2 import DictLoader, Environment __path__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__), '..')) @@ -18,11 +18,7 @@ class Module: return '<{} ({})>'.format(self.title, self.name) def asdict(self): - return { - 'name': self.name, - 'title': self.title, - 'automodule_options': self.automodule_options, - } + return {'name': self.name, 'title': self.title, 'automodule_options': self.automodule_options} def get_path(self): return os.path.join(__path__, apidoc_root, *self.name.split('.')) + '.rst' @@ -45,9 +41,9 @@ def underlined_filter(txt, chr): env = Environment( - loader=DictLoader({ - 'module': - ''' + loader=DictLoader( + { + 'module': ''' {{ (':mod:`'~title~' <'~name~'>`') | underlined('=') }} .. currentmodule:: {{ name }} @@ -56,8 +52,12 @@ env = Environment( .. automodule:: {{ name }} {% for opt in automodule_options %} :{{ opt }}:{{ "\n" }}{% endfor %} - ''' [1:-1] + '\n' - }) + '''[ + 1:-1 + ] + + '\n' + } + ) ) env.filters['underlined'] = underlined_filter diff --git a/bonobo/__init__.py b/bonobo/__init__.py index e3e46b7..97d5443 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -31,9 +31,7 @@ def _repr_html_(): '
{}
' ' 
{}
' '' - ).format( - __logo__, '
'.join(get_versions(all=True)) - ) + ).format(__logo__, '
'.join(get_versions(all=True))) del sys diff --git a/bonobo/_api.py b/bonobo/_api.py index 5d38224..3baf5a1 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -8,11 +8,11 @@ to another is maximal. """ from bonobo.execution.strategies import create_strategy -from bonobo.nodes import __all__ as _all_nodes from bonobo.nodes import * +from bonobo.nodes import __all__ as _all_nodes from bonobo.structs import Graph from bonobo.util.api import ApiHelper -from bonobo.util.environ import parse_args, get_argument_parser +from bonobo.util.environ import get_argument_parser, parse_args __all__ = [] @@ -44,14 +44,17 @@ def run(graph, *, plugins=None, services=None, strategy=None): plugins = plugins or [] from bonobo import settings + settings.check() if not settings.QUIET.get(): # pragma: no cover if _is_interactive_console(): import mondrian + mondrian.setup(excepthook=True) from bonobo.plugins.console import ConsoleOutputPlugin + if ConsoleOutputPlugin not in plugins: plugins.append(ConsoleOutputPlugin) @@ -60,6 +63,7 @@ def run(graph, *, plugins=None, services=None, strategy=None): from bonobo.contrib.jupyter import JupyterOutputPlugin except ImportError: import logging + logging.warning( 'Failed to load jupyter widget. Easiest way is to install the optional "jupyter" ' 'dependencies with «pip install bonobo[jupyter]», but you can also install a specific ' @@ -70,6 +74,7 @@ def run(graph, *, plugins=None, services=None, strategy=None): plugins.append(JupyterOutputPlugin) import logging + logging.getLogger().setLevel(settings.LOGGING_LEVEL.get()) strategy = create_strategy(strategy) return strategy.execute(graph, plugins=plugins, services=services) @@ -158,6 +163,7 @@ api.register_group( def _is_interactive_console(): import sys + return sys.stdout.isatty() @@ -172,6 +178,7 @@ def _is_jupyter_notebook(): def get_examples_path(*pathsegments): import os import pathlib + return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments)) diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index a482b53..fb6de85 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -42,6 +42,7 @@ def entrypoint(args=None): logger.exception('Error while loading command {}.'.format(ext.name)) from stevedore import ExtensionManager + mgr = ExtensionManager(namespace='bonobo.commands') mgr.map(register_extension) diff --git a/bonobo/commands/base.py b/bonobo/commands/base.py index da2967f..4f15daa 100644 --- a/bonobo/commands/base.py +++ b/bonobo/commands/base.py @@ -41,6 +41,7 @@ class BaseGraphCommand(BaseCommand): Base class for CLI commands that depends on a graph definition, either from a file or from a module. """ + required = True handler = None diff --git a/bonobo/commands/convert.py b/bonobo/commands/convert.py index d04e97b..c20ac40 100644 --- a/bonobo/commands/convert.py +++ b/bonobo/commands/convert.py @@ -1,7 +1,7 @@ import bonobo from bonobo.commands import BaseCommand from bonobo.registry import READER, WRITER, default_registry -from bonobo.util.resolvers import _resolve_transformations, _resolve_options +from bonobo.util.resolvers import _resolve_options, _resolve_transformations class ConvertCommand(BaseCommand): @@ -11,21 +11,14 @@ class ConvertCommand(BaseCommand): parser.add_argument( '--' + READER, '-r', - help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.' + help='Choose the reader factory if it cannot be detected from extension, or if detection is wrong.', ) parser.add_argument( '--' + WRITER, '-w', - help= - 'Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).' - ) - parser.add_argument( - '--limit', - '-l', - type=int, - help='Adds a Limit() after the reader instance.', - default=None, + help='Choose the writer factory if it cannot be detected from extension, or if detection is wrong (use - for console pretty print).', ) + parser.add_argument('--limit', '-l', type=int, help='Adds a Limit() after the reader instance.', default=None) parser.add_argument( '--transformation', '-t', @@ -56,16 +49,16 @@ class ConvertCommand(BaseCommand): ) def handle( - self, - input_filename, - output_filename, - reader=None, - reader_option=None, - writer=None, - writer_option=None, - option=None, - limit=None, - transformation=None, + self, + input_filename, + output_filename, + reader=None, + reader_option=None, + writer=None, + writer_option=None, + option=None, + limit=None, + transformation=None, ): reader_factory = default_registry.get_reader_factory_for(input_filename, format=reader) reader_kwargs = _resolve_options((option or []) + (reader_option or [])) @@ -75,13 +68,13 @@ class ConvertCommand(BaseCommand): writer_args = () else: writer_factory = default_registry.get_writer_factory_for(output_filename, format=writer) - writer_args = (output_filename, ) + writer_args = (output_filename,) writer_kwargs = _resolve_options((option or []) + (writer_option or [])) transformations = () if limit: - transformations += (bonobo.Limit(limit), ) + transformations += (bonobo.Limit(limit),) transformations += _resolve_transformations(transformation) @@ -92,8 +85,4 @@ class ConvertCommand(BaseCommand): writer_factory(*writer_args, **writer_kwargs), ) - return bonobo.run( - graph, services={ - 'fs': bonobo.open_fs(), - } - ) + return bonobo.run(graph, services={'fs': bonobo.open_fs()}) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index ce76bfc..d0bf5cb 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -19,6 +19,7 @@ class RunCommand(BaseGraphCommand): def parse_options(self, *, quiet=False, verbose=False, install=False, **options): from bonobo import settings + settings.QUIET.set_if_true(quiet) settings.DEBUG.set_if_true(verbose) self.install = install @@ -65,4 +66,5 @@ def _install_requirements(requirements): # python interpreter. pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources) import site + importlib.reload(site) diff --git a/bonobo/commands/version.py b/bonobo/commands/version.py index 5ca2311..e0153cc 100644 --- a/bonobo/commands/version.py +++ b/bonobo/commands/version.py @@ -32,10 +32,11 @@ class VersionCommand(BaseCommand): def _format_version(mod, *, name=None, quiet=False): from bonobo.util.pkgs import bonobo_packages + args = { 'name': name or mod.__name__, 'version': mod.__version__, - 'location': bonobo_packages[name or mod.__name__].location + 'location': bonobo_packages[name or mod.__name__].location, } if not quiet: diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index 50f41fc..a894a84 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -1,9 +1,7 @@ from bonobo.errors import AbstractError -from bonobo.util import isoption, iscontextprocessor, sortedlist, get_name +from bonobo.util import get_name, iscontextprocessor, isoption, sortedlist -__all__ = [ - 'Configurable', -] +__all__ = ['Configurable'] get_creation_counter = lambda v: v._creation_counter @@ -64,10 +62,7 @@ class ConfigurableMeta(type): return cls.__processors_cache def __repr__(self): - return ' '.join(( - ' 1 else '', ', '.join(map(repr, sorted(extraneous))) + cls.__name__, + len(extraneous), + 's' if len(extraneous) > 1 else '', + ', '.join(map(repr, sorted(extraneous))), ) ) @@ -167,8 +164,10 @@ class Configurable(metaclass=ConfigurableMeta): if _final: raise TypeError( '{}() missing {} required option{}: {}.'.format( - cls.__name__, len(missing), 's' - if len(missing) > 1 else '', ', '.join(map(repr, sorted(missing))) + cls.__name__, + len(missing), + 's' if len(missing) > 1 else '', + ', '.join(map(repr, sorted(missing))), ) ) return PartiallyConfigured(cls, *args, **kwargs) diff --git a/bonobo/config/functools.py b/bonobo/config/functools.py index 7beaf6f..4bf9c31 100644 --- a/bonobo/config/functools.py +++ b/bonobo/config/functools.py @@ -1,5 +1,4 @@ import functools - import itertools diff --git a/bonobo/config/options.py b/bonobo/config/options.py index cdf9411..5f8b566 100644 --- a/bonobo/config/options.py +++ b/bonobo/config/options.py @@ -113,8 +113,9 @@ class RemovedOption(Option): def clean(self, value): if value != self.value: raise ValueError( - 'Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.'. - format(self.name, self.value) + 'Removed options cannot change value, {!r} must now be {!r} (and you should remove setting the value explicitely, as it is deprecated and will be removed quite soon.'.format( + self.name, self.value + ) ) return self.value @@ -195,9 +196,7 @@ class Method(Option): if not callable(value): raise TypeError( 'Option {!r} ({}) is expecting a callable value, got {!r} object: {!r}.'.format( - self.name, - type(self).__name__, - type(value).__name__, value + self.name, type(self).__name__, type(value).__name__, value ) ) inst._options_values[self.name] = self.type(value) if self.type else value diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index c8a2ddf..7083484 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -101,15 +101,17 @@ class ContextCurrifier: try: bound = self._bind(_input) except TypeError as exc: - raise UnrecoverableTypeError(( - 'Input of {wrapped!r} does not bind to the node signature.\n' - 'Args: {args}\n' - 'Input: {input}\n' - 'Kwargs: {kwargs}\n' - 'Signature: {sig}' - ).format( - wrapped=self.wrapped, args=self.args, input=_input, kwargs=self.kwargs, sig=signature(self.wrapped) - )) from exc + raise UnrecoverableTypeError( + ( + 'Input of {wrapped!r} does not bind to the node signature.\n' + 'Args: {args}\n' + 'Input: {input}\n' + 'Kwargs: {kwargs}\n' + 'Signature: {sig}' + ).format( + wrapped=self.wrapped, args=self.args, input=_input, kwargs=self.kwargs, sig=signature(self.wrapped) + ) + ) from exc return self.wrapped(*bound.args, **bound.kwargs) def setup(self, *context): diff --git a/bonobo/config/services.py b/bonobo/config/services.py index 282d88f..5a9f392 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -112,10 +112,12 @@ def create_container(services=None, factory=Container): if not 'fs' in container: import bonobo + container.setdefault('fs', bonobo.open_fs()) if not 'http' in container: import requests + container.setdefault('http', requests) return container @@ -139,6 +141,7 @@ class Exclusive(ContextDecorator): ensure that. """ + _locks = {} def __init__(self, wrapped): diff --git a/bonobo/contrib/django/__init__.py b/bonobo/contrib/django/__init__.py index d159eea..52b0154 100644 --- a/bonobo/contrib/django/__init__.py +++ b/bonobo/contrib/django/__init__.py @@ -9,7 +9,4 @@ This module contains all tools for Bonobo and Django to interract nicely. from .utils import create_or_update from .commands import ETLCommand -__all__ = [ - 'ETLCommand', - 'create_or_update', -] +__all__ = ['ETLCommand', 'create_or_update'] diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py index 828c3a3..1fb38c3 100644 --- a/bonobo/contrib/django/commands.py +++ b/bonobo/contrib/django/commands.py @@ -1,13 +1,14 @@ from logging import getLogger from types import GeneratorType +from colorama import Back, Fore, Style +from django.core.management import BaseCommand +from django.core.management.base import OutputWrapper +from mondrian import term + import bonobo from bonobo.plugins.console import ConsoleOutputPlugin from bonobo.util.term import CLEAR_EOL -from colorama import Fore, Back, Style -from django.core.management import BaseCommand -from django.core.management.base import OutputWrapper -from mondrian import term from .utils import create_or_update @@ -55,7 +56,7 @@ class ETLCommand(BaseCommand): graph_coll = self.get_graph(*args, **options) if not isinstance(graph_coll, GeneratorType): - graph_coll = (graph_coll, ) + graph_coll = (graph_coll,) for i, graph in enumerate(graph_coll): assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.' diff --git a/bonobo/contrib/google/__init__.py b/bonobo/contrib/google/__init__.py index 74d4612..4138401 100644 --- a/bonobo/contrib/google/__init__.py +++ b/bonobo/contrib/google/__init__.py @@ -41,14 +41,14 @@ def get_credentials(*, scopes): return credentials -def get_google_spreadsheets_api_client(scopes=('https://www.googleapis.com/auth/spreadsheets', )): +def get_google_spreadsheets_api_client(scopes=('https://www.googleapis.com/auth/spreadsheets',)): credentials = get_credentials(scopes=scopes) http = credentials.authorize(httplib2.Http()) discoveryUrl = 'https://sheets.googleapis.com/$discovery/rest?version=v4' return discovery.build('sheets', 'v4', http=http, discoveryServiceUrl=discoveryUrl, cache_discovery=False) -def get_google_people_api_client(scopes=('https://www.googleapis.com/auth/contacts', )): +def get_google_people_api_client(scopes=('https://www.googleapis.com/auth/contacts',)): credentials = get_credentials(scopes=scopes) http = credentials.authorize(httplib2.Http()) discoveryUrl = 'https://people.googleapis.com/$discovery/rest?version=v1' diff --git a/bonobo/contrib/jupyter/__init__.py b/bonobo/contrib/jupyter/__init__.py index 49242be..faf30ac 100644 --- a/bonobo/contrib/jupyter/__init__.py +++ b/bonobo/contrib/jupyter/__init__.py @@ -5,6 +5,4 @@ def _jupyter_nbextension_paths(): return [{'section': 'notebook', 'src': 'static', 'dest': 'bonobo-jupyter', 'require': 'bonobo-jupyter/extension'}] -__all__ = [ - 'JupyterOutputPlugin', -] +__all__ = ['JupyterOutputPlugin'] diff --git a/bonobo/contrib/opendatasoft/__init__.py b/bonobo/contrib/opendatasoft/__init__.py index 5144e59..5020114 100644 --- a/bonobo/contrib/opendatasoft/__init__.py +++ b/bonobo/contrib/opendatasoft/__init__.py @@ -44,15 +44,9 @@ class OpenDataSoftAPI(Configurable): break for row in records: - yield { - **row.get('fields', {}), - 'geometry': row.get('geometry', {}), - 'recordid': row.get('recordid'), - } + yield {**row.get('fields', {}), 'geometry': row.get('geometry', {}), 'recordid': row.get('recordid')} start += self.rows -__all__ = [ - 'OpenDataSoftAPI', -] +__all__ = ['OpenDataSoftAPI'] diff --git a/bonobo/errors.py b/bonobo/errors.py index 173ce40..ed41181 100644 --- a/bonobo/errors.py +++ b/bonobo/errors.py @@ -16,10 +16,7 @@ class InactiveWritableError(InactiveIOError): class ValidationError(RuntimeError): def __init__(self, inst, message): super(ValidationError, self).__init__( - 'Validation error in {class_name}: {message}'.format( - class_name=type(inst).__name__, - message=message, - ) + 'Validation error in {class_name}: {message}'.format(class_name=type(inst).__name__, message=message) ) @@ -42,8 +39,7 @@ class AbstractError(UnrecoverableError, NotImplementedError): def __init__(self, method): super().__init__( 'Call to abstract method {class_name}.{method_name}(...): missing implementation.'.format( - class_name=get_name(method.__self__), - method_name=get_name(method), + class_name=get_name(method.__self__), method_name=get_name(method) ) ) diff --git a/bonobo/examples/__init__.py b/bonobo/examples/__init__.py index ec68fc5..0498c4d 100644 --- a/bonobo/examples/__init__.py +++ b/bonobo/examples/__init__.py @@ -4,19 +4,9 @@ import bonobo def get_argument_parser(parser=None): parser = bonobo.get_argument_parser(parser=parser) + parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.') parser.add_argument( - '--limit', - '-l', - type=int, - default=None, - help='If set, limits the number of processed lines.' - ) - parser.add_argument( - '--print', - '-p', - action='store_true', - default=False, - help='If set, pretty prints before writing to output file.' + '--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.' ) return parser @@ -26,7 +16,4 @@ def get_graph_options(options): _limit = options.pop('limit', None) _print = options.pop('print', False) - return { - '_limit': (bonobo.Limit(_limit), ) if _limit else (), - '_print': (bonobo.PrettyPrinter(), ) if _print else (), - } + return {'_limit': (bonobo.Limit(_limit),) if _limit else (), '_print': (bonobo.PrettyPrinter(),) if _print else ()} diff --git a/bonobo/examples/clock.py b/bonobo/examples/clock.py index 1977cba..9d6a875 100644 --- a/bonobo/examples/clock.py +++ b/bonobo/examples/clock.py @@ -1,7 +1,8 @@ -import bonobo import datetime import time +import bonobo + def extract(): """Placeholder, change, rename, remove... """ @@ -13,10 +14,7 @@ def extract(): def get_graph(): graph = bonobo.Graph() - graph.add_chain( - extract, - print, - ) + graph.add_chain(extract, print) return graph diff --git a/bonobo/examples/datasets/__main__.py b/bonobo/examples/datasets/__main__.py index 91f702d..555c02e 100644 --- a/bonobo/examples/datasets/__main__.py +++ b/bonobo/examples/datasets/__main__.py @@ -4,26 +4,18 @@ import bonobo from bonobo import examples from bonobo.examples.datasets.coffeeshops import get_graph as get_coffeeshops_graph from bonobo.examples.datasets.fablabs import get_graph as get_fablabs_graph -from bonobo.examples.datasets.services import get_services, get_datasets_dir, get_minor_version +from bonobo.examples.datasets.services import get_datasets_dir, get_minor_version, get_services -graph_factories = { - 'coffeeshops': get_coffeeshops_graph, - 'fablabs': get_fablabs_graph, -} +graph_factories = {'coffeeshops': get_coffeeshops_graph, 'fablabs': get_fablabs_graph} if __name__ == '__main__': parser = examples.get_argument_parser() - parser.add_argument( - '--target', '-t', choices=graph_factories.keys(), nargs='+' - ) + parser.add_argument('--target', '-t', choices=graph_factories.keys(), nargs='+') parser.add_argument('--sync', action='store_true', default=False) with bonobo.parse_args(parser) as options: graph_options = examples.get_graph_options(options) - graph_names = list( - options['target'] - if options['target'] else sorted(graph_factories.keys()) - ) + graph_names = list(options['target'] if options['target'] else sorted(graph_factories.keys())) # Create a graph with all requested subgraphs graph = bonobo.Graph() @@ -43,18 +35,9 @@ if __name__ == '__main__': for filename in files: local_path = os.path.join(root, filename) relative_path = os.path.relpath(local_path, local_dir) - s3_path = os.path.join( - get_minor_version(), relative_path - ) + s3_path = os.path.join(get_minor_version(), relative_path) try: - s3.head_object( - Bucket='bonobo-examples', Key=s3_path - ) + s3.head_object(Bucket='bonobo-examples', Key=s3_path) except: - s3.upload_file( - local_path, - 'bonobo-examples', - s3_path, - ExtraArgs={'ACL': 'public-read'} - ) + s3.upload_file(local_path, 'bonobo-examples', s3_path, ExtraArgs={'ACL': 'public-read'}) diff --git a/bonobo/examples/datasets/coffeeshops.py b/bonobo/examples/datasets/coffeeshops.py index fdca3cd..e599dd5 100644 --- a/bonobo/examples/datasets/coffeeshops.py +++ b/bonobo/examples/datasets/coffeeshops.py @@ -13,48 +13,26 @@ def get_graph(graph=None, *, _limit=(), _print=()): graph = graph or bonobo.Graph() producer = graph.add_chain( - ODSReader( - dataset='liste-des-cafes-a-un-euro', - netloc='opendata.paris.fr' - ), + ODSReader(dataset='liste-des-cafes-a-un-euro', netloc='opendata.paris.fr'), *_limit, bonobo.UnpackItems(0), - bonobo.Rename( - name='nom_du_cafe', - address='adresse', - zipcode='arrondissement' - ), + bonobo.Rename(name='nom_du_cafe', address='adresse', zipcode='arrondissement'), bonobo.Format(city='Paris', country='France'), - bonobo.OrderFields( - [ - 'name', 'address', 'zipcode', 'city', 'country', - 'geometry', 'geoloc' - ] - ), + bonobo.OrderFields(['name', 'address', 'zipcode', 'city', 'country', 'geometry', 'geoloc']), *_print, ) # Comma separated values. graph.add_chain( - bonobo.CsvWriter( - 'coffeeshops.csv', - fields=['name', 'address', 'zipcode', 'city'], - delimiter=',' - ), + bonobo.CsvWriter('coffeeshops.csv', fields=['name', 'address', 'zipcode', 'city'], delimiter=','), _input=producer.output, ) # Standard JSON - graph.add_chain( - bonobo.JsonWriter(path='coffeeshops.json'), - _input=producer.output, - ) + graph.add_chain(bonobo.JsonWriter(path='coffeeshops.json'), _input=producer.output) # Line-delimited JSON - graph.add_chain( - bonobo.LdjsonWriter(path='coffeeshops.ldjson'), - _input=producer.output, - ) + graph.add_chain(bonobo.LdjsonWriter(path='coffeeshops.ldjson'), _input=producer.output) return graph @@ -63,7 +41,4 @@ if __name__ == '__main__': parser = examples.get_argument_parser() with bonobo.parse_args(parser) as options: - bonobo.run( - get_graph(**examples.get_graph_options(options)), - services=get_services() - ) + bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services()) diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py index 9d2f4ed..dca6ae4 100644 --- a/bonobo/examples/datasets/fablabs.py +++ b/bonobo/examples/datasets/fablabs.py @@ -24,9 +24,7 @@ from bonobo.examples.datasets.services import get_services try: import pycountry except ImportError as exc: - raise ImportError( - 'You must install package "pycountry" to run this example.' - ) from exc + raise ImportError('You must install package "pycountry" to run this example.') from exc API_DATASET = 'fablabs@public-us' ROWS = 100 @@ -39,12 +37,8 @@ def _getlink(x): def normalize(row): result = { **row, - 'links': - list(filter(None, map(_getlink, json.loads(row.get('links'))))), - 'country': - pycountry.countries.get( - alpha_2=row.get('country_code', '').upper() - ).name, + 'links': list(filter(None, map(_getlink, json.loads(row.get('links'))))), + 'country': pycountry.countries.get(alpha_2=row.get('country_code', '').upper()).name, } return result @@ -66,7 +60,4 @@ if __name__ == '__main__': parser = examples.get_argument_parser() with bonobo.parse_args(parser) as options: - bonobo.run( - get_graph(**examples.get_graph_options(options)), - services=get_services() - ) + bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services()) diff --git a/bonobo/examples/datasets/services.py b/bonobo/examples/datasets/services.py index 6412156..a1b9ebb 100644 --- a/bonobo/examples/datasets/services.py +++ b/bonobo/examples/datasets/services.py @@ -9,9 +9,7 @@ def get_minor_version(): def get_datasets_dir(*dirs): home_dir = os.path.expanduser('~') - target_dir = os.path.join( - home_dir, '.cache/bonobo', get_minor_version(), *dirs - ) + target_dir = os.path.join(home_dir, '.cache/bonobo', get_minor_version(), *dirs) os.makedirs(target_dir, exist_ok=True) return target_dir diff --git a/bonobo/examples/files/_services.py b/bonobo/examples/files/_services.py index 825e39d..68bc598 100644 --- a/bonobo/examples/files/_services.py +++ b/bonobo/examples/files/_services.py @@ -2,7 +2,4 @@ from bonobo import get_examples_path, open_fs def get_services(): - return { - 'fs': open_fs(get_examples_path()), - 'fs.output': open_fs(), - } + return {'fs': open_fs(get_examples_path()), 'fs.output': open_fs()} diff --git a/bonobo/examples/files/csv_handlers.py b/bonobo/examples/files/csv_handlers.py index acc6189..9df225b 100644 --- a/bonobo/examples/files/csv_handlers.py +++ b/bonobo/examples/files/csv_handlers.py @@ -5,8 +5,8 @@ from bonobo.examples.files._services import get_services def get_graph(*, _limit=None, _print=False): return bonobo.Graph( bonobo.CsvReader('datasets/coffeeshops.txt'), - *((bonobo.Limit(_limit), ) if _limit else ()), - *((bonobo.PrettyPrinter(), ) if _print else ()), + *((bonobo.Limit(_limit),) if _limit else ()), + *((bonobo.PrettyPrinter(),) if _print else ()), bonobo.CsvWriter('coffeeshops.csv', fs='fs.output') ) @@ -14,23 +14,10 @@ def get_graph(*, _limit=None, _print=False): if __name__ == '__main__': parser = bonobo.get_argument_parser() + parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.') parser.add_argument( - '--limit', - '-l', - type=int, - default=None, - help='If set, limits the number of processed lines.' - ) - parser.add_argument( - '--print', - '-p', - action='store_true', - default=False, - help='If set, pretty prints before writing to output file.' + '--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.' ) with bonobo.parse_args(parser) as options: - bonobo.run( - get_graph(_limit=options['limit'], _print=options['print']), - services=get_services() - ) + bonobo.run(get_graph(_limit=options['limit'], _print=options['print']), services=get_services()) diff --git a/bonobo/examples/files/json_handlers.py b/bonobo/examples/files/json_handlers.py index 819a8fd..ab79894 100644 --- a/bonobo/examples/files/json_handlers.py +++ b/bonobo/examples/files/json_handlers.py @@ -5,22 +5,13 @@ from bonobo.examples.files._services import get_services def get_graph(*, _limit=None, _print=False): graph = bonobo.Graph() - trunk = graph.add_chain( - bonobo.JsonReader('datasets/theaters.json'), - *((bonobo.Limit(_limit), ) if _limit else ()), - ) + trunk = graph.add_chain(bonobo.JsonReader('datasets/theaters.json'), *((bonobo.Limit(_limit),) if _limit else ())) if _print: graph.add_chain(bonobo.PrettyPrinter(), _input=trunk.output) - graph.add_chain( - bonobo.JsonWriter('theaters.json', fs='fs.output'), - _input=trunk.output - ) - graph.add_chain( - bonobo.LdjsonWriter('theaters.ldjson', fs='fs.output'), - _input=trunk.output - ) + graph.add_chain(bonobo.JsonWriter('theaters.json', fs='fs.output'), _input=trunk.output) + graph.add_chain(bonobo.LdjsonWriter('theaters.ldjson', fs='fs.output'), _input=trunk.output) return graph @@ -28,23 +19,10 @@ def get_graph(*, _limit=None, _print=False): if __name__ == '__main__': parser = bonobo.get_argument_parser() + parser.add_argument('--limit', '-l', type=int, default=None, help='If set, limits the number of processed lines.') parser.add_argument( - '--limit', - '-l', - type=int, - default=None, - help='If set, limits the number of processed lines.' - ) - parser.add_argument( - '--print', - '-p', - action='store_true', - default=False, - help='If set, pretty prints before writing to output file.' + '--print', '-p', action='store_true', default=False, help='If set, pretty prints before writing to output file.' ) with bonobo.parse_args(parser) as options: - bonobo.run( - get_graph(_limit=options['limit'], _print=options['print']), - services=get_services() - ) + bonobo.run(get_graph(_limit=options['limit'], _print=options['print']), services=get_services()) diff --git a/bonobo/examples/files/pickle_handlers.py b/bonobo/examples/files/pickle_handlers.py index 0b90955..2ba0acd 100644 --- a/bonobo/examples/files/pickle_handlers.py +++ b/bonobo/examples/files/pickle_handlers.py @@ -35,9 +35,7 @@ from bonobo import examples def cleanse_sms(category, sms): if category == 'spam': - sms_clean = '**MARKED AS SPAM** ' + sms[0:50] + ( - '...' if len(sms) > 50 else '' - ) + sms_clean = '**MARKED AS SPAM** ' + sms[0:50] + ('...' if len(sms) > 50 else '') elif category == 'ham': sms_clean = sms else: @@ -62,16 +60,11 @@ def get_graph(*, _limit=(), _print=()): def get_services(): from ._services import get_services - return { - **get_services(), 'fs': - TarFS(bonobo.get_examples_path('datasets/spam.tgz')) - } + + return {**get_services(), 'fs': TarFS(bonobo.get_examples_path('datasets/spam.tgz'))} if __name__ == '__main__': parser = examples.get_argument_parser() with bonobo.parse_args(parser) as options: - bonobo.run( - get_graph(**examples.get_graph_options(options)), - services=get_services() - ) + bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services()) diff --git a/bonobo/examples/files/text_handlers.py b/bonobo/examples/files/text_handlers.py index 2e91227..d23e6c3 100644 --- a/bonobo/examples/files/text_handlers.py +++ b/bonobo/examples/files/text_handlers.py @@ -23,7 +23,4 @@ def get_graph(*, _limit=(), _print=()): if __name__ == '__main__': parser = examples.get_argument_parser() with bonobo.parse_args(parser) as options: - bonobo.run( - get_graph(**examples.get_graph_options(options)), - services=get_services() - ) + bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services()) diff --git a/bonobo/execution/contexts/__init__.py b/bonobo/execution/contexts/__init__.py index 41e811e..db12704 100644 --- a/bonobo/execution/contexts/__init__.py +++ b/bonobo/execution/contexts/__init__.py @@ -9,8 +9,4 @@ from bonobo.execution.contexts.graph import GraphExecutionContext from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.execution.contexts.plugin import PluginExecutionContext -__all__ = [ - 'GraphExecutionContext', - 'NodeExecutionContext', - 'PluginExecutionContext', -] +__all__ = ['GraphExecutionContext', 'NodeExecutionContext', 'PluginExecutionContext'] diff --git a/bonobo/execution/contexts/graph.py b/bonobo/execution/contexts/graph.py index a6559a3..b41d16f 100644 --- a/bonobo/execution/contexts/graph.py +++ b/bonobo/execution/contexts/graph.py @@ -1,12 +1,13 @@ from functools import partial from time import sleep +from whistle import EventDispatcher + from bonobo.config import create_container from bonobo.constants import BEGIN, END from bonobo.execution import events from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.execution.contexts.plugin import PluginExecutionContext -from whistle import EventDispatcher class GraphExecutionContext: diff --git a/bonobo/execution/contexts/node.py b/bonobo/execution/contexts/node.py index 316d9b8..1bc84c4 100644 --- a/bonobo/execution/contexts/node.py +++ b/bonobo/execution/contexts/node.py @@ -7,11 +7,11 @@ from types import GeneratorType from bonobo.config import create_container from bonobo.config.processors import ContextCurrifier -from bonobo.constants import NOT_MODIFIED, BEGIN, END, TICK_PERIOD, Token, Flag, INHERIT +from bonobo.constants import BEGIN, END, INHERIT, NOT_MODIFIED, TICK_PERIOD, Flag, Token from bonobo.errors import InactiveReadableError, UnrecoverableError, UnrecoverableTypeError from bonobo.execution.contexts.base import BaseContext from bonobo.structs.inputs import Input -from bonobo.util import get_name, isconfigurabletype, ensure_tuple +from bonobo.util import ensure_tuple, get_name, isconfigurabletype from bonobo.util.bags import BagType from bonobo.util.statistics import WithStatistics @@ -105,10 +105,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): except Empty: sleep(TICK_PERIOD) # XXX: How do we determine this constant? continue - except ( - NotImplementedError, - UnrecoverableError, - ): + except (NotImplementedError, UnrecoverableError): self.fatal(sys.exc_info()) # exit loop except Exception: # pylint: disable=broad-except self.error(sys.exc_info()) # does not exit loop diff --git a/bonobo/execution/strategies/__init__.py b/bonobo/execution/strategies/__init__.py index 7eacbcf..90a13de 100644 --- a/bonobo/execution/strategies/__init__.py +++ b/bonobo/execution/strategies/__init__.py @@ -9,9 +9,7 @@ at home if you want to give it a shot. from bonobo.execution.strategies.executor import ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy from bonobo.execution.strategies.naive import NaiveStrategy -__all__ = [ - 'create_strategy', -] +__all__ = ['create_strategy'] STRATEGIES = { 'naive': NaiveStrategy, diff --git a/bonobo/execution/strategies/base.py b/bonobo/execution/strategies/base.py index 0a8d2a5..cc64367 100644 --- a/bonobo/execution/strategies/base.py +++ b/bonobo/execution/strategies/base.py @@ -6,6 +6,7 @@ class Strategy: Base class for execution strategies. """ + GraphExecutionContextType = GraphExecutionContext def __init__(self, GraphExecutionContextType=None): diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 9708996..665eedc 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -3,14 +3,15 @@ import html import itertools import pprint +from mondrian import term + from bonobo import settings -from bonobo.config import Configurable, Option, Method, use_raw_input, use_context, use_no_input +from bonobo.config import Configurable, Method, Option, use_context, use_no_input, use_raw_input from bonobo.config.functools import transformation_factory from bonobo.config.processors import ContextProcessor, use_context_processor from bonobo.constants import NOT_MODIFIED from bonobo.util.objects import ValueHolder from bonobo.util.term import CLEAR_EOL -from mondrian import term __all__ = [ 'FixedWindow', @@ -43,6 +44,7 @@ class Limit(Configurable): TODO: simplify into a closure building factory? """ + limit = Option(positional=True, default=10) @ContextProcessor @@ -69,7 +71,7 @@ def Tee(f): def _shorten(s, w): if w and len(s) > w: - s = s[0:w - 3] + '...' + s = s[0 : w - 3] + '...' return s @@ -80,17 +82,19 @@ class PrettyPrinter(Configurable): required=False, __doc__=''' If set, truncates the output values longer than this to this width. - ''' + ''', ) filter = Method( - default= - (lambda self, index, key, value: (value is not None) and (not isinstance(key, str) or not key.startswith('_'))), + default=( + lambda self, index, key, value: (value is not None) + and (not isinstance(key, str) or not key.startswith('_')) + ), __doc__=''' A filter that determine what to print. Default is to ignore any key starting with an underscore and none values. - ''' + ''', ) @ContextProcessor @@ -99,6 +103,7 @@ class PrettyPrinter(Configurable): yield context if context._jupyter_html is not None: from IPython.display import display, HTML + display(HTML('\n'.join([''] + context._jupyter_html + ['
']))) def __call__(self, context, *args, **kwargs): @@ -153,16 +158,11 @@ class PrettyPrinter(Configurable): if not context._jupyter_html: context._jupyter_html = [ '', - *map('{}'.format, map(html.escape, map(str, - context.get_input_fields() or range(len(args))))), + *map('{}'.format, map(html.escape, map(str, context.get_input_fields() or range(len(args))))), '', ] - context._jupyter_html += [ - '', - *map('{}'.format, map(html.escape, map(repr, args))), - '', - ] + context._jupyter_html += ['', *map('{}'.format, map(html.escape, map(repr, args))), ''] @use_no_input diff --git a/bonobo/nodes/filter.py b/bonobo/nodes/filter.py index 0e0026f..88e3101 100644 --- a/bonobo/nodes/filter.py +++ b/bonobo/nodes/filter.py @@ -1,6 +1,5 @@ -from bonobo.constants import NOT_MODIFIED - from bonobo.config import Configurable, Method +from bonobo.constants import NOT_MODIFIED class Filter(Configurable): diff --git a/bonobo/nodes/io/base.py b/bonobo/nodes/io/base.py index d2cca60..10ca463 100644 --- a/bonobo/nodes/io/base.py +++ b/bonobo/nodes/io/base.py @@ -13,22 +13,39 @@ class FileHandler(Configurable): """ path = Option( - str, required=True, positional=True, __doc__=''' + str, + required=True, + positional=True, + __doc__=''' Path to use within the provided filesystem. - ''' + ''', ) # type: str - eol = Option(str, default='\n', __doc__=''' + eol = Option( + str, + default='\n', + __doc__=''' Character to use as line separator. - ''') # type: str - mode = Option(str, __doc__=''' + ''', + ) # type: str + mode = Option( + str, + __doc__=''' What mode to use for open() call. - ''') # type: str - encoding = Option(str, default='utf-8', __doc__=''' + ''', + ) # type: str + encoding = Option( + str, + default='utf-8', + __doc__=''' Encoding. - ''') # type: str - fs = Service('fs', __doc__=''' + ''', + ) # type: str + fs = Service( + 'fs', + __doc__=''' The filesystem instance to use. - ''') # type: str + ''', + ) # type: str @ContextProcessor def file(self, context, *, fs): diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index d7d03fb..c6fbdb6 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -1,6 +1,6 @@ import csv -from bonobo.config import Option, use_raw_input, use_context +from bonobo.config import Option, use_context, use_raw_input from bonobo.config.options import Method, RenamedOption from bonobo.constants import NOT_MODIFIED from bonobo.nodes.io.base import FileHandler @@ -62,7 +62,7 @@ class CsvReader(FileReader, CsvHandler): default=0, __doc__=''' If set and greater than zero, the reader will skip this amount of lines. - ''' + ''', ) @Method( @@ -72,7 +72,7 @@ class CsvReader(FileReader, CsvHandler): iterable. Defaults to builtin csv.reader(...), but can be overriden to fit your special needs. - ''' + ''', ) def reader_factory(self, file): return csv.reader(file, **self.get_dialect_kwargs()) diff --git a/bonobo/nodes/io/file.py b/bonobo/nodes/io/file.py index 7cbf410..b63db4f 100644 --- a/bonobo/nodes/io/file.py +++ b/bonobo/nodes/io/file.py @@ -1,4 +1,4 @@ -from bonobo.config import Option, ContextProcessor, use_context +from bonobo.config import ContextProcessor, Option, use_context from bonobo.constants import NOT_MODIFIED from bonobo.errors import UnrecoverableError from bonobo.nodes.io.base import FileHandler, Reader, Writer @@ -12,9 +12,13 @@ class FileReader(Reader, FileHandler): present. Extending it is usually the right way to create more specific file readers (like json, csv, etc.) """ - mode = Option(str, default='r', __doc__=''' + mode = Option( + str, + default='r', + __doc__=''' What mode to use for open() call. - ''') # type: str + ''', + ) # type: str output_fields = Option( ensure_tuple, @@ -22,14 +26,14 @@ class FileReader(Reader, FileHandler): __doc__=''' Specify the field names of output lines. Mutually exclusive with "output_type". - ''' + ''', ) output_type = Option( required=False, __doc__=''' Specify the type of output lines. Mutually exclusive with "output_fields". - ''' + ''', ) @ContextProcessor @@ -72,9 +76,13 @@ class FileWriter(Writer, FileHandler): usually the right way to create more specific file writers (like json, csv, etc.) """ - mode = Option(str, default='w+', __doc__=''' + mode = Option( + str, + default='w+', + __doc__=''' What mode to use for open() call. - ''') # type: str + ''', + ) # type: str def write(self, file, context, line, *, fs): """ diff --git a/bonobo/plugins/console.py b/bonobo/plugins/console.py index 69a044c..6de10c0 100644 --- a/bonobo/plugins/console.py +++ b/bonobo/plugins/console.py @@ -1,8 +1,9 @@ import io import sys -from contextlib import redirect_stdout, redirect_stderr +from contextlib import redirect_stderr, redirect_stdout -from colorama import Style, Fore, init as initialize_colorama_output_wrappers +from colorama import Fore, Style +from colorama import init as initialize_colorama_output_wrappers from bonobo import settings from bonobo.execution import events @@ -34,7 +35,7 @@ class ConsoleOutputPlugin(Plugin): isatty = False # Whether we're on windows, or a real operating system. - iswindows = (sys.platform == 'win32') + iswindows = sys.platform == 'win32' def __init__(self): self.isatty = self._stdout.isatty() @@ -95,27 +96,32 @@ class ConsoleOutputPlugin(Plugin): liveliness_color = alive_color if node.alive else dead_color liveliness_prefix = ' {}{}{} '.format(liveliness_color, node.status, Style.RESET_ALL) - _line = ''.join(( - liveliness_prefix, - node.name, - name_suffix, - ' ', - node.get_statistics_as_string(), - ' ', - node.get_flags_as_string(), - Style.RESET_ALL, - ' ', - )) + _line = ''.join( + ( + liveliness_prefix, + node.name, + name_suffix, + ' ', + node.get_statistics_as_string(), + ' ', + node.get_flags_as_string(), + Style.RESET_ALL, + ' ', + ) + ) print(prefix + _line + CLEAR_EOL, file=self._stderr) if append: # todo handle multiline print( - ''.join(( - ' `-> ', ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), - CLEAR_EOL - )), - file=self._stderr + ''.join( + ( + ' `-> ', + ' '.join('{}{}{}: {}'.format(Style.BRIGHT, k, Style.RESET_ALL, v) for k, v in append), + CLEAR_EOL, + ) + ), + file=self._stderr, ) t_cnt += 1 @@ -128,16 +134,17 @@ class ConsoleOutputPlugin(Plugin): if self.counter % 10 and self._append_cache: append = self._append_cache else: - self._append_cache = append = (('Memory', '{0:.2f} Mb'.format(memory_usage())), - # ('Total time', '{0} s'.format(execution_time(harness))), - ) + self._append_cache = append = ( + ('Memory', '{0:.2f} Mb'.format(memory_usage())), + # ('Total time', '{0} s'.format(execution_time(harness))), + ) else: append = () self.write(context, prefix=self.prefix, append=append, rewind=rewind) self.counter += 1 -class IOBuffer(): +class IOBuffer: """ The role of IOBuffer is to overcome the problem of multiple threads wanting to write to stdout at the same time. It works a bit like a videogame: there are two buffers, one that is used to write, and one which is used to read from. @@ -164,5 +171,6 @@ class IOBuffer(): def memory_usage(): import os, psutil + process = psutil.Process(os.getpid()) - return process.memory_info()[0] / float(2**20) + return process.memory_info()[0] / float(2 ** 20) diff --git a/bonobo/registry.py b/bonobo/registry.py index be8d47b..4a03d6a 100644 --- a/bonobo/registry.py +++ b/bonobo/registry.py @@ -1,8 +1,7 @@ import mimetypes - import os -from bonobo import JsonReader, CsvReader, PickleReader, FileReader, FileWriter, PickleWriter, CsvWriter, JsonWriter +from bonobo import CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter, PickleReader, PickleWriter FILETYPE_CSV = 'text/csv' FILETYPE_JSON = 'application/json' diff --git a/bonobo/settings.py b/bonobo/settings.py index 799ba3d..9c1cfb8 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -1,5 +1,4 @@ import logging - import os from bonobo.errors import ValidationError @@ -92,17 +91,14 @@ LOGGING_LEVEL = Setting( 'LOGGING_LEVEL', formatter=logging._checkLevel, validator=logging._checkLevel, - default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO + default=lambda: logging.DEBUG if DEBUG.get() else logging.INFO, ) # Input/Output format for transformations IOFORMAT_ARG0 = 'arg0' IOFORMAT_KWARGS = 'kwargs' -IOFORMATS = { - IOFORMAT_ARG0, - IOFORMAT_KWARGS, -} +IOFORMATS = {IOFORMAT_ARG0, IOFORMAT_KWARGS} IOFORMAT = Setting('IOFORMAT', default=IOFORMAT_KWARGS, validator=IOFORMATS.__contains__) diff --git a/bonobo/structs/__init__.py b/bonobo/structs/__init__.py index ba640c9..17e5fac 100644 --- a/bonobo/structs/__init__.py +++ b/bonobo/structs/__init__.py @@ -1,5 +1,3 @@ from bonobo.structs.graphs import Graph -__all__ = [ - 'Graph', -] +__all__ = ['Graph'] diff --git a/bonobo/structs/graphs.py b/bonobo/structs/graphs.py index 292d7b2..280f2ff 100644 --- a/bonobo/structs/graphs.py +++ b/bonobo/structs/graphs.py @@ -3,11 +3,12 @@ import json from collections import namedtuple from copy import copy -from bonobo.constants import BEGIN -from bonobo.util import get_name from graphviz import ExecutableNotFound from graphviz.dot import Digraph +from bonobo.constants import BEGIN +from bonobo.util import get_name + GraphRange = namedtuple('GraphRange', ['graph', 'input', 'output']) @@ -15,6 +16,7 @@ class Graph: """ Represents a directed graph of nodes. """ + name = '' def __init__(self, *chain): diff --git a/bonobo/util/__init__.py b/bonobo/util/__init__.py index 894d053..3701142 100644 --- a/bonobo/util/__init__.py +++ b/bonobo/util/__init__.py @@ -16,7 +16,7 @@ from bonobo.util.inspect import ( istuple, istype, ) -from bonobo.util.objects import (get_name, get_attribute_or_create, ValueHolder) +from bonobo.util.objects import get_name, get_attribute_or_create, ValueHolder # Bonobo's util API __all__ = [ diff --git a/bonobo/util/api.py b/bonobo/util/api.py index 7bfba02..ed0213c 100644 --- a/bonobo/util/api.py +++ b/bonobo/util/api.py @@ -12,14 +12,14 @@ class ApiHelper: if graph: # This function must comply to the "graph" API interface, meaning it can bahave like bonobo.run. from inspect import signature + parameters = list(signature(x).parameters) required_parameters = {'plugins', 'services', 'strategy'} - assert len(parameters - ) > 0 and parameters[0] == 'graph', 'First parameter of a graph api function must be "graph".' - assert required_parameters.intersection( - parameters - ) == required_parameters, 'Graph api functions must define the following parameters: ' + ', '.join( - sorted(required_parameters) + assert ( + len(parameters) > 0 and parameters[0] == 'graph' + ), 'First parameter of a graph api function must be "graph".' + assert required_parameters.intersection(parameters) == required_parameters, ( + 'Graph api functions must define the following parameters: ' + ', '.join(sorted(required_parameters)) ) self.__all__.append(get_name(x)) diff --git a/bonobo/util/bags.py b/bonobo/util/bags.py index fd31d06..49a7035 100644 --- a/bonobo/util/bags.py +++ b/bonobo/util/bags.py @@ -73,7 +73,9 @@ class {typename}(tuple): _field_template = '''\ {name} = _property(_itemgetter({index:d}), doc={doc!r}) -'''.strip('\n') +'''.strip( + '\n' +) _reserved = frozenset( ['_', '_cls', '_attrs', '_fields', 'get', '_asdict', '_replace', '_make', 'self', '_self', 'tuple'] + dir(tuple) @@ -150,16 +152,19 @@ def BagType(typename, fields, *, verbose=False, module=None): attrs=attrs, num_fields=len(fields), arg_list=repr(attrs).replace("'", "")[1:-1], - repr_fmt=', '.join(('%r' if isinstance(fields[index], int) else '{name}=%r').format(name=name) - for index, name in enumerate(attrs)), + repr_fmt=', '.join( + ('%r' if isinstance(fields[index], int) else '{name}=%r').format(name=name) + for index, name in enumerate(attrs) + ), field_defs='\n'.join( _field_template.format( index=index, name=name, - doc='Alias for ' + - ('field #{}'.format(index) if isinstance(fields[index], int) else repr(fields[index])) - ) for index, name in enumerate(attrs) - ) + doc='Alias for ' + + ('field #{}'.format(index) if isinstance(fields[index], int) else repr(fields[index])), + ) + for index, name in enumerate(attrs) + ), ) # Execute the template string in a temporary namespace and support diff --git a/bonobo/util/collections.py b/bonobo/util/collections.py index 3e46738..de706f4 100644 --- a/bonobo/util/collections.py +++ b/bonobo/util/collections.py @@ -26,7 +26,7 @@ def ensure_tuple(tuple_or_mixed, *, cls=tuple): if isinstance(tuple_or_mixed, tuple): return tuple.__new__(cls, tuple_or_mixed) - return tuple.__new__(cls, (tuple_or_mixed, )) + return tuple.__new__(cls, (tuple_or_mixed,)) def cast(type_): diff --git a/bonobo/util/compat.py b/bonobo/util/compat.py index 4a62742..292d1a0 100644 --- a/bonobo/util/compat.py +++ b/bonobo/util/compat.py @@ -9,7 +9,7 @@ def deprecated_alias(alias, func): warnings.warn( "Call to deprecated function alias {}, use {} instead.".format(alias, func.__name__), category=DeprecationWarning, - stacklevel=2 + stacklevel=2, ) warnings.simplefilter('default', DeprecationWarning) # reset filter return func(*args, **kwargs) diff --git a/bonobo/util/environ.py b/bonobo/util/environ.py index b344d29..6cdb76f 100644 --- a/bonobo/util/environ.py +++ b/bonobo/util/environ.py @@ -58,6 +58,7 @@ def get_argument_parser(parser=None): """ if parser is None: import argparse + parser = argparse.ArgumentParser() # Store globally to be able to warn the user about the fact he's probably wrong not to pass a parser to @@ -94,6 +95,7 @@ def parse_args(mixed=None): ) # use the api from bonobo namespace, in case a command patched it. import bonobo + mixed = bonobo.get_argument_parser() if isinstance(mixed, argparse.ArgumentParser): diff --git a/bonobo/util/inspect.py b/bonobo/util/inspect.py index a7f27c4..e24e59e 100644 --- a/bonobo/util/inspect.py +++ b/bonobo/util/inspect.py @@ -9,6 +9,7 @@ def isconfigurable(mixed): :return: bool """ from bonobo.config.configurables import Configurable + return isinstance(mixed, Configurable) @@ -47,6 +48,7 @@ def isoption(mixed): """ from bonobo.config.options import Option + return isinstance(mixed, Option) @@ -58,6 +60,7 @@ def ismethod(mixed): :return: bool """ from bonobo.config.options import Method + return isinstance(mixed, Method) @@ -69,6 +72,7 @@ def iscontextprocessor(x): :return: bool """ from bonobo.config.processors import ContextProcessor + return isinstance(x, ContextProcessor) @@ -102,15 +106,7 @@ def istuple(mixed): return isinstance(mixed, tuple) -ConfigurableInspection = namedtuple( - 'ConfigurableInspection', [ - 'type', - 'instance', - 'options', - 'processors', - 'partial', - ] -) +ConfigurableInspection = namedtuple('ConfigurableInspection', ['type', 'instance', 'options', 'processors', 'partial']) ConfigurableInspection.__enter__ = lambda self: self ConfigurableInspection.__exit__ = lambda *exc_details: None @@ -141,10 +137,4 @@ def inspect_node(mixed, *, _partial=None): 'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.' ) - return ConfigurableInspection( - typ, - inst, - list(typ.__options__), - list(typ.__processors__), - _partial, - ) + return ConfigurableInspection(typ, inst, list(typ.__options__), list(typ.__processors__), _partial) diff --git a/bonobo/util/objects.py b/bonobo/util/objects.py index f3ffa5e..209f4db 100644 --- a/bonobo/util/objects.py +++ b/bonobo/util/objects.py @@ -142,10 +142,10 @@ class ValueHolder: return divmod(other, self._value) def __pow__(self, other): - return self._value**other + return self._value ** other def __rpow__(self, other): - return other**self._value + return other ** self._value def __ipow__(self, other): self._value **= other diff --git a/bonobo/util/term.py b/bonobo/util/term.py index 0920b22..2fa02ce 100644 --- a/bonobo/util/term.py +++ b/bonobo/util/term.py @@ -1,2 +1,2 @@ CLEAR_EOL = '\033[0K' -MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n) \ No newline at end of file +MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n) diff --git a/bonobo/util/testing.py b/bonobo/util/testing.py index 8000e89..e34ed74 100644 --- a/bonobo/util/testing.py +++ b/bonobo/util/testing.py @@ -4,12 +4,12 @@ import io import os import runpy import sys -from contextlib import contextmanager, redirect_stdout, redirect_stderr +from contextlib import contextmanager, redirect_stderr, redirect_stdout from unittest.mock import patch import pytest -from bonobo import open_fs, __main__, get_examples_path +from bonobo import __main__, get_examples_path, open_fs from bonobo.commands import entrypoint from bonobo.constants import Token from bonobo.execution.contexts.graph import GraphExecutionContext @@ -112,19 +112,13 @@ def runner_module(args): all_runners = pytest.mark.parametrize('runner', [runner_entrypoint, runner_module]) all_environ_targets = pytest.mark.parametrize( - 'target', [ - (get_examples_path('environ.py'), ), - ( - '-m', - 'bonobo.examples.environ', - ), - ] + 'target', [(get_examples_path('environ.py'),), ('-m', 'bonobo.examples.environ')] ) @all_runners @all_environ_targets -class EnvironmentTestCase(): +class EnvironmentTestCase: def run_quiet(self, runner, *args): return runner('run', '--quiet', *args) @@ -216,12 +210,12 @@ class ReaderTest(ConfigurableNodeTest): self.tmpdir = tmpdir def get_create_args(self, *args): - return (self.filename, ) + args + return (self.filename,) + args def test_customizable_output_type_transform_not_a_type(self): context = self.NodeExecutionContextType( self.create(*self.get_create_args(), output_type=str.upper, **self.get_create_kwargs()), - services=self.services + services=self.services, ) with pytest.raises(TypeError): context.start() @@ -229,9 +223,9 @@ class ReaderTest(ConfigurableNodeTest): def test_customizable_output_type_transform_not_a_tuple(self): context = self.NodeExecutionContextType( self.create( - *self.get_create_args(), output_type=type('UpperString', (str, ), {}), **self.get_create_kwargs() + *self.get_create_args(), output_type=type('UpperString', (str,), {}), **self.get_create_kwargs() ), - services=self.services + services=self.services, ) with pytest.raises(TypeError): context.start() @@ -256,7 +250,7 @@ class WriterTest(ConfigurableNodeTest): self.tmpdir = tmpdir def get_create_args(self, *args): - return (self.filename, ) + args + return (self.filename,) + args def readlines(self): with self.fs.open(self.filename) as fp: diff --git a/docs/_templates/alabaster/support.py b/docs/_templates/alabaster/support.py index 9147b52..6788588 100644 --- a/docs/_templates/alabaster/support.py +++ b/docs/_templates/alabaster/support.py @@ -1,8 +1,9 @@ # flake8: noqa from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal +from pygments.token import ( + Comment, Error, Generic, Keyword, Literal, Name, Number, Operator, Other, Punctuation, String, Whitespace +) # Originally based on FlaskyStyle which was based on 'tango'. @@ -12,7 +13,7 @@ class Alabaster(Style): styles = { # No corresponding class for the following: - #Text: "", # class: '' + # Text: "", # class: '' Whitespace: "underline #f8f8f8", # class: 'w' Error: "#a40000 border:#ef2929", # class: 'err' Other: "#000000", # class 'x' @@ -28,7 +29,6 @@ class Alabaster(Style): Operator: "#582800", # class: 'o' Operator.Word: "bold #004461", # class: 'ow' - like keywords Punctuation: "bold #000000", # class: 'p' - # because special names such as Name.Class, Name.Function, etc. # are not recognized as such later in the parsing, we choose them # to look the same as ordinary variables. diff --git a/docs/conf.py b/docs/conf.py index 877ebd7..73e6ec3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,10 +5,11 @@ import datetime import os import sys +import bonobo + sys.path.insert(0, os.path.abspath('..')) sys.path.insert(0, os.path.abspath('_themes')) -import bonobo # -- General configuration ------------------------------------------------ @@ -63,11 +64,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] autoclass_content = 'both' autodoc_member_order = 'groupwise' -autodoc_default_flags = [ - 'members', - 'undoc-members', - 'show-inheritance', -] +autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance'] add_module_names = False pygments_style = 'sphinx' @@ -112,7 +109,7 @@ html_sidebars = { 'sourcelink.html', 'searchbox.html', 'sidebarinfos.html', - ] + ], } html_theme_path = ['_themes'] @@ -137,15 +134,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -154,9 +148,7 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual'), -] +latex_documents = [(master_doc, 'Bonobo.tex', 'Bonobo Documentation', 'Romain Dorgueil', 'manual')] # -- Options for manual page output --------------------------------------- @@ -171,9 +163,14 @@ man_pages = [(master_doc, 'bonobo', 'Bonobo Documentation', [author], 1)] # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, 'Bonobo', 'Bonobo Documentation', author, 'Bonobo', 'One line description of project.', - 'Miscellaneous' - ), + master_doc, + 'Bonobo', + 'Bonobo Documentation', + author, + 'Bonobo', + 'One line description of project.', + 'Miscellaneous', + ) ] # -- Options for Epub output ---------------------------------------------- @@ -209,4 +206,6 @@ rst_epilog = """ .. |longversion| replace:: v.{version} -""".format(version=version, ) +""".format( + version=version +) diff --git a/requirements-dev.txt b/requirements-dev.txt index aa9fab3..366ecf1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,10 @@ -e .[dev] -r requirements.txt -alabaster==0.7.10 +alabaster==0.7.11 arrow==0.12.1 +atomicwrites==1.1.5 attrs==18.1.0 -babel==2.5.3 +babel==2.6.0 binaryornot==0.4.4 certifi==2018.4.16 chardet==3.0.4 @@ -12,29 +13,29 @@ cookiecutter==1.5.1 coverage==4.5.1 docutils==0.14 future==0.16.0 -idna==2.6 +idna==2.7 imagesize==1.0.0 jinja2-time==0.2.0 jinja2==2.10 markupsafe==1.0 -more-itertools==4.1.0 +more-itertools==4.3.0 packaging==17.1 -pluggy==0.6.0 +pathlib2==2.3.2 +pluggy==0.7.1 poyo==0.4.1 -py==1.5.3 +py==1.5.4 pygments==2.2.0 pyparsing==2.2.0 pytest-cov==2.5.1 -pytest-timeout==1.2.1 -pytest==3.5.1 +pytest-timeout==1.3.1 +pytest==3.7.1 python-dateutil==2.7.3 -pytz==2018.4 -requests==2.18.4 +pytz==2018.5 +requests==2.19.1 six==1.11.0 snowballstemmer==1.2.1 sphinx-sitemap==0.2 -sphinx==1.7.4 -sphinxcontrib-websupport==1.0.1 -urllib3==1.22 +sphinx==1.7.6 +sphinxcontrib-websupport==1.1.0 +urllib3==1.23 whichcraft==0.4.1 -yapf==0.22.0 diff --git a/requirements-docker.txt b/requirements-docker.txt index f25e570..68469e2 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -5,26 +5,26 @@ bonobo-docker==0.6.0 certifi==2018.4.16 chardet==3.0.4 colorama==0.3.9 -docker-pycreds==0.2.3 +docker-pycreds==0.3.0 docker==2.7.0 -fs==2.0.23 -graphviz==0.8.3 -idna==2.6 +fs==2.0.27 +graphviz==0.8.4 +idna==2.7 jinja2==2.10 markupsafe==1.0 mondrian==0.7.0 packaging==17.1 -pbr==4.0.3 -psutil==5.4.5 +pbr==4.2.0 +psutil==5.4.6 pyparsing==2.2.0 python-slugify==1.2.5 -pytz==2018.4 -requests==2.18.4 +pytz==2018.5 +requests==2.19.1 semantic-version==2.6.0 six==1.11.0 -stevedore==1.28.0 +stevedore==1.29.0 typing==3.6.4 unidecode==1.0.22 -urllib3==1.22 -websocket-client==0.47.0 +urllib3==1.23 +websocket-client==0.48.0 whistle==1.0.1 diff --git a/requirements-jupyter.txt b/requirements-jupyter.txt index 0d3908b..f96b3b6 100644 --- a/requirements-jupyter.txt +++ b/requirements-jupyter.txt @@ -8,9 +8,9 @@ entrypoints==0.2.3 html5lib==1.0.1 ipykernel==4.8.2 ipython-genutils==0.2.0 -ipython==6.4.0 +ipython==6.5.0 ipywidgets==6.0.1 -jedi==0.12.0 +jedi==0.12.1 jinja2==2.10 jsonschema==2.6.0 jupyter-client==5.2.3 @@ -21,23 +21,24 @@ markupsafe==1.0 mistune==0.8.3 nbconvert==5.3.1 nbformat==4.4.0 -notebook==5.5.0 +notebook==5.6.0 pandocfilters==1.4.2 -parso==0.2.0 -pexpect==4.5.0 +parso==0.3.1 +pexpect==4.6.0 pickleshare==0.7.4 +prometheus-client==0.3.1 prompt-toolkit==1.0.15 -ptyprocess==0.5.2 +ptyprocess==0.6.0 pygments==2.2.0 python-dateutil==2.7.3 -pyzmq==17.0.0 +pyzmq==17.1.2 qtconsole==4.3.1 send2trash==1.5.0 simplegeneric==0.8.1 six==1.11.0 terminado==0.8.1 testpath==0.3.1 -tornado==5.0.2 +tornado==5.1 traitlets==4.3.2 wcwidth==0.1.7 webencodings==0.5.1 diff --git a/requirements-sqlalchemy.txt b/requirements-sqlalchemy.txt index 3d8083d..e1fedde 100644 --- a/requirements-sqlalchemy.txt +++ b/requirements-sqlalchemy.txt @@ -5,23 +5,23 @@ bonobo-sqlalchemy==0.6.0 certifi==2018.4.16 chardet==3.0.4 colorama==0.3.9 -fs==2.0.23 -graphviz==0.8.3 -idna==2.6 +fs==2.0.27 +graphviz==0.8.4 +idna==2.7 jinja2==2.10 markupsafe==1.0 mondrian==0.7.0 packaging==17.1 -pbr==4.0.3 -psutil==5.4.5 +pbr==4.2.0 +psutil==5.4.6 pyparsing==2.2.0 python-slugify==1.2.5 -pytz==2018.4 -requests==2.18.4 +pytz==2018.5 +requests==2.19.1 six==1.11.0 -sqlalchemy==1.2.7 -stevedore==1.28.0 +sqlalchemy==1.2.10 +stevedore==1.29.0 typing==3.6.4 unidecode==1.0.22 -urllib3==1.22 +urllib3==1.23 whistle==1.0.1 diff --git a/requirements.txt b/requirements.txt index 89a7968..3a5402c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,22 +3,22 @@ appdirs==1.4.3 certifi==2018.4.16 chardet==3.0.4 colorama==0.3.9 -fs==2.0.23 -graphviz==0.8.3 -idna==2.6 +fs==2.0.27 +graphviz==0.8.4 +idna==2.7 jinja2==2.10 markupsafe==1.0 mondrian==0.7.0 packaging==17.1 -pbr==4.0.3 -psutil==5.4.5 +pbr==4.2.0 +psutil==5.4.6 pyparsing==2.2.0 python-slugify==1.2.5 -pytz==2018.4 -requests==2.18.4 +pytz==2018.5 +requests==2.19.1 six==1.11.0 -stevedore==1.28.0 +stevedore==1.29.0 typing==3.6.4 unidecode==1.0.22 -urllib3==1.22 +urllib3==1.23 whistle==1.0.1 diff --git a/setup.py b/setup.py index b5334d2..f732536 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,12 @@ -# Generated by Medikit 0.6.1 on 2018-05-21. +# Generated by Medikit 0.6.3 on 2018-08-11. # All changes will be overriden. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate. -from setuptools import setup, find_packages from codecs import open from os import path +from setuptools import find_packages, setup + here = path.abspath(path.dirname(__file__)) # Py3 compatibility hacks, borrowed from IPython. @@ -44,14 +45,17 @@ else: setup( author='Romain Dorgueil', author_email='romain@dorgueil.net', - data_files=[( - 'share/jupyter/nbextensions/bonobo-jupyter', [ - 'bonobo/contrib/jupyter/static/extension.js', 'bonobo/contrib/jupyter/static/index.js', - 'bonobo/contrib/jupyter/static/index.js.map' - ] - )], - description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' - 'python 3.5+.'), + data_files=[ + ( + 'share/jupyter/nbextensions/bonobo-jupyter', + [ + 'bonobo/contrib/jupyter/static/extension.js', + 'bonobo/contrib/jupyter/static/index.js', + 'bonobo/contrib/jupyter/static/index.js.map', + ], + ) + ], + description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'python 3.5+.'), license='Apache License, Version 2.0', name='bonobo', version=version, @@ -60,26 +64,42 @@ setup( packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, install_requires=[ - 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', 'mondrian (~= 0.7)', 'packaging (~= 17.0)', - 'psutil (~= 5.4)', 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)', 'stevedore (~= 1.27)', 'whistle (~= 1.0)' + 'fs (~= 2.0)', + 'graphviz (>= 0.8, < 0.9)', + 'jinja2 (~= 2.9)', + 'mondrian (~= 0.7)', + 'packaging (~= 17.0)', + 'psutil (~= 5.4)', + 'python-slugify (~= 1.2.0)', + 'requests (~= 2.0)', + 'stevedore (~= 1.27)', + 'whistle (~= 1.0)', ], extras_require={ 'dev': [ - 'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)', 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)', - 'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)', 'sphinx-sitemap (>= 0.2, < 0.3)', 'yapf' + 'cookiecutter (>= 1.5, < 1.6)', + 'coverage (~= 4.4)', + 'pytest (~= 3.4)', + 'pytest-cov (~= 2.5)', + 'pytest-timeout (>= 1, < 2)', + 'sphinx (~= 1.7)', + 'sphinx-sitemap (>= 0.2, < 0.3)', ], 'docker': ['bonobo-docker (~= 0.6.0a1)'], 'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'], - 'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'] + 'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'], }, entry_points={ 'bonobo.commands': [ - 'convert = bonobo.commands.convert:ConvertCommand', 'download = bonobo.commands.download:DownloadCommand', - 'examples = bonobo.commands.examples:ExamplesCommand', 'init = bonobo.commands.init:InitCommand', - 'inspect = bonobo.commands.inspect:InspectCommand', 'run = bonobo.commands.run:RunCommand', - 'version = bonobo.commands.version:VersionCommand' + 'convert = bonobo.commands.convert:ConvertCommand', + 'download = bonobo.commands.download:DownloadCommand', + 'examples = bonobo.commands.examples:ExamplesCommand', + 'init = bonobo.commands.init:InitCommand', + 'inspect = bonobo.commands.inspect:InspectCommand', + 'run = bonobo.commands.run:RunCommand', + 'version = bonobo.commands.version:VersionCommand', ], - 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] + 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], }, url='https://www.bonobo-project.org/', download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), diff --git a/tests/commands/test_clibasics.py b/tests/commands/test_clibasics.py index 1fc292b..4800ec8 100644 --- a/tests/commands/test_clibasics.py +++ b/tests/commands/test_clibasics.py @@ -9,17 +9,11 @@ def test_entrypoint(): for command in pkg_resources.iter_entry_points('bonobo.commands'): commands[command.name] = command - assert not { - 'convert', - 'init', - 'inspect', - 'run', - 'version', - }.difference(set(commands)) + assert not {'convert', 'init', 'inspect', 'run', 'version'}.difference(set(commands)) @all_runners def test_no_command(runner): _, err, exc = runner(catch_errors=True) assert type(exc) == SystemExit - assert 'error: the following arguments are required: command' in err \ No newline at end of file + assert 'error: the following arguments are required: command' in err diff --git a/tests/commands/test_download.py b/tests/commands/test_download.py index 83b0ef4..1ce1069 100644 --- a/tests/commands/test_download.py +++ b/tests/commands/test_download.py @@ -27,8 +27,9 @@ def test_download_works_for_examples(runner): fout = io.BytesIO() fout.close = lambda: None - with patch('bonobo.commands.download._open_url') as mock_open_url, \ - patch('bonobo.commands.download.open') as mock_open: + with patch('bonobo.commands.download._open_url') as mock_open_url, patch( + 'bonobo.commands.download.open' + ) as mock_open: mock_open_url.return_value = MockResponse() mock_open.return_value = fout runner('download', 'examples/datasets/coffeeshops.txt') @@ -41,4 +42,4 @@ def test_download_works_for_examples(runner): @all_runners def test_download_fails_non_example(runner): with pytest.raises(ValueError): - runner('download', 'something/entirely/different.txt') \ No newline at end of file + runner('download', 'something/entirely/different.txt') diff --git a/tests/commands/test_run_environ.py b/tests/commands/test_run_environ.py index 3fee6e1..e541c4d 100644 --- a/tests/commands/test_run_environ.py +++ b/tests/commands/test_run_environ.py @@ -6,21 +6,14 @@ from bonobo.util.testing import EnvironmentTestCase @pytest.fixture def env1(tmpdir): env_file = tmpdir.join('.env_one') - env_file.write('\n'.join(( - 'SECRET=unknown', - 'PASSWORD=sweet', - 'PATH=first', - ))) + env_file.write('\n'.join(('SECRET=unknown', 'PASSWORD=sweet', 'PATH=first'))) return str(env_file) @pytest.fixture def env2(tmpdir): env_file = tmpdir.join('.env_two') - env_file.write('\n'.join(( - 'PASSWORD=bitter', - "PATH='second'", - ))) + env_file.write('\n'.join(('PASSWORD=bitter', "PATH='second'"))) return str(env_file) @@ -71,7 +64,15 @@ class TestEnvFileCombinations(EnvironmentTestCase): def test_run_with_both_env_files_then_overrides(self, runner, target, env1, env2): env = self.run_environ( - runner, *target, '--default-env-file', env1, '--env-file', env2, '--env', 'PASSWORD=mine', '--env', + runner, + *target, + '--default-env-file', + env1, + '--env-file', + env2, + '--env', + 'PASSWORD=mine', + '--env', 'SECRET=s3cr3t' ) assert env.get('SECRET') == 's3cr3t' diff --git a/tests/commands/test_version.py b/tests/commands/test_version.py index 1ee893f..d25fd1e 100644 --- a/tests/commands/test_version.py +++ b/tests/commands/test_version.py @@ -17,4 +17,4 @@ def test_version(runner): out, err = runner('version', '-qq') out = out.strip() assert not out.startswith('bonobo ') - assert __version__ in out \ No newline at end of file + assert __version__ in out diff --git a/tests/config/test_methods.py b/tests/config/test_methods.py index b5c3a45..9faae13 100644 --- a/tests/config/test_methods.py +++ b/tests/config/test_methods.py @@ -50,10 +50,7 @@ def test_define_with_decorator(): calls = [] def my_handler(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) Concrete = MethodBasedConfigurable(my_handler) @@ -77,10 +74,7 @@ def test_late_binding_method_decoration(): @MethodBasedConfigurable(foo='foo') def Concrete(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) assert callable(Concrete.handler) t = Concrete(bar='baz') @@ -95,10 +89,7 @@ def test_define_with_argument(): calls = [] def concrete_handler(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) t = MethodBasedConfigurable(concrete_handler, 'foo', bar='baz') assert callable(t.handler) @@ -112,10 +103,7 @@ def test_define_with_inheritance(): class Inheriting(MethodBasedConfigurable): def handler(self, *args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) t = Inheriting('foo', bar='baz') assert callable(t.handler) @@ -132,10 +120,7 @@ def test_inheritance_then_decorate(): @Inheriting def Concrete(*args, **kwargs): - calls.append(( - args, - kwargs, - )) + calls.append((args, kwargs)) assert callable(Concrete.handler) t = Concrete('foo', bar='baz') diff --git a/tests/config/test_methods_partial.py b/tests/config/test_methods_partial.py index e45f11d..2e5ad52 100644 --- a/tests/config/test_methods_partial.py +++ b/tests/config/test_methods_partial.py @@ -40,7 +40,7 @@ def test_partial(): assert len(ci.options) == 4 assert len(ci.processors) == 1 assert ci.partial - assert ci.partial[0] == (f1, ) + assert ci.partial[0] == (f1,) assert not len(ci.partial[1]) # instanciate a more complete partial instance ... @@ -53,10 +53,7 @@ def test_partial(): assert len(ci.options) == 4 assert len(ci.processors) == 1 assert ci.partial - assert ci.partial[0] == ( - f1, - f2, - ) + assert ci.partial[0] == (f1, f2) assert not len(ci.partial[1]) c = C('foo') diff --git a/tests/config/test_processors.py b/tests/config/test_processors.py index c6b3c9a..f735c95 100644 --- a/tests/config/test_processors.py +++ b/tests/config/test_processors.py @@ -1,7 +1,7 @@ from operator import attrgetter from bonobo.config import Configurable -from bonobo.config.processors import ContextProcessor, resolve_processors, ContextCurrifier, use_context_processor +from bonobo.config.processors import ContextCurrifier, ContextProcessor, resolve_processors, use_context_processor class CP1(Configurable): diff --git a/tests/config/test_services.py b/tests/config/test_services.py index 078832c..5671e90 100644 --- a/tests/config/test_services.py +++ b/tests/config/test_services.py @@ -4,11 +4,11 @@ import time import pytest from bonobo.config import Configurable, Container, Exclusive, Service, use -from bonobo.config.services import validate_service_name, create_container +from bonobo.config.services import create_container, validate_service_name from bonobo.util import get_name -class PrinterInterface(): +class PrinterInterface: def print(self, *args): raise NotImplementedError() @@ -21,14 +21,11 @@ class ConcretePrinter(PrinterInterface): return ';'.join((self.prefix, *args)) -SERVICES = Container( - printer0=ConcretePrinter(prefix='0'), - printer1=ConcretePrinter(prefix='1'), -) +SERVICES = Container(printer0=ConcretePrinter(prefix='0'), printer1=ConcretePrinter(prefix='1')) class MyServiceDependantConfigurable(Configurable): - printer = Service(PrinterInterface, ) + printer = Service(PrinterInterface) def __call__(self, *args, printer: PrinterInterface): return printer.print(*args) @@ -80,7 +77,7 @@ def test_exclusive(): vcr.append(' '.join((prefix, str(i)))) time.sleep(0.05) - threads = [threading.Thread(target=record, args=(str(i), )) for i in range(5)] + threads = [threading.Thread(target=record, args=(str(i),)) for i in range(5)] for thread in threads: thread.start() @@ -90,8 +87,32 @@ def test_exclusive(): thread.join() assert vcr.tape == [ - 'hello', '0 0', '0 1', '0 2', '0 3', '0 4', '1 0', '1 1', '1 2', '1 3', '1 4', '2 0', '2 1', '2 2', '2 3', - '2 4', '3 0', '3 1', '3 2', '3 3', '3 4', '4 0', '4 1', '4 2', '4 3', '4 4' + 'hello', + '0 0', + '0 1', + '0 2', + '0 3', + '0 4', + '1 0', + '1 1', + '1 2', + '1 3', + '1 4', + '2 0', + '2 1', + '2 2', + '2 3', + '2 4', + '3 0', + '3 1', + '3 2', + '3 3', + '3 4', + '4 0', + '4 1', + '4 2', + '4 3', + '4 4', ] @@ -118,10 +139,7 @@ def test_create_container_empty_values(services): def test_create_container_override(): - c = create_container({ - 'http': 'http', - 'fs': 'fs', - }) + c = create_container({'http': 'http', 'fs': 'fs'}) assert len(c) == 2 assert 'fs' in c and c['fs'] == 'fs' assert 'http' in c and c['http'] == 'http' diff --git a/tests/execution/contexts/test_node.py b/tests/execution/contexts/test_node.py index 5af665d..2b5c378 100644 --- a/tests/execution/contexts/test_node.py +++ b/tests/execution/contexts/test_node.py @@ -3,10 +3,10 @@ from unittest.mock import MagicMock import pytest from bonobo import Graph -from bonobo.constants import EMPTY, NOT_MODIFIED, INHERIT +from bonobo.constants import EMPTY, INHERIT, NOT_MODIFIED from bonobo.execution.contexts.node import NodeExecutionContext, split_token from bonobo.execution.strategies import NaiveStrategy -from bonobo.util.testing import BufferingNodeExecutionContext, BufferingGraphExecutionContext +from bonobo.util.testing import BufferingGraphExecutionContext, BufferingNodeExecutionContext def test_node_string(): @@ -18,7 +18,7 @@ def test_node_string(): output = context.get_buffer() assert len(output) == 1 - assert output[0] == ('foo', ) + assert output[0] == ('foo',) def g(): yield 'foo' @@ -29,8 +29,8 @@ def test_node_string(): output = context.get_buffer() assert len(output) == 2 - assert output[0] == ('foo', ) - assert output[1] == ('bar', ) + assert output[0] == ('foo',) + assert output[1] == ('bar',) def test_node_bytes(): @@ -42,7 +42,7 @@ def test_node_bytes(): output = context.get_buffer() assert len(output) == 1 - assert output[0] == (b'foo', ) + assert output[0] == (b'foo',) def g(): yield b'foo' @@ -53,8 +53,8 @@ def test_node_bytes(): output = context.get_buffer() assert len(output) == 2 - assert output[0] == (b'foo', ) - assert output[1] == (b'bar', ) + assert output[0] == (b'foo',) + assert output[1] == (b'bar',) def test_node_dict(): @@ -65,7 +65,7 @@ def test_node_dict(): context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 1 - assert output[0] == ({'id': 1, 'name': 'foo'}, ) + assert output[0] == ({'id': 1, 'name': 'foo'},) def g(): yield {'id': 1, 'name': 'foo'} @@ -75,8 +75,8 @@ def test_node_dict(): context.write_sync(EMPTY) output = context.get_buffer() assert len(output) == 2 - assert output[0] == ({'id': 1, 'name': 'foo'}, ) - assert output[1] == ({'id': 2, 'name': 'bar'}, ) + assert output[0] == ({'id': 1, 'name': 'foo'},) + assert output[1] == ({'id': 2, 'name': 'bar'},) def test_node_dict_chained(): @@ -93,7 +93,7 @@ def test_node_dict_chained(): output = context.get_buffer() assert len(output) == 1 - assert output[0] == ({'id': 1, 'name': 'FOO'}, ) + assert output[0] == ({'id': 1, 'name': 'FOO'},) def g(): yield {'id': 1, 'name': 'foo'} @@ -104,8 +104,8 @@ def test_node_dict_chained(): output = context.get_buffer() assert len(output) == 2 - assert output[0] == ({'id': 1, 'name': 'FOO'}, ) - assert output[1] == ({'id': 2, 'name': 'BAR'}, ) + assert output[0] == ({'id': 1, 'name': 'FOO'},) + assert output[1] == ({'id': 2, 'name': 'BAR'},) def test_node_tuple(): @@ -229,7 +229,7 @@ def test_node_lifecycle_with_kill(): def test_split_token(): assert split_token(('foo', 'bar')) == (set(), ('foo', 'bar')) assert split_token(()) == (set(), ()) - assert split_token('') == (set(), ('', )) + assert split_token('') == (set(), ('',)) def test_split_token_duplicate(): @@ -249,10 +249,10 @@ def test_split_token_not_modified(): with pytest.raises(ValueError): split_token((INHERIT, NOT_MODIFIED)) assert split_token(NOT_MODIFIED) == ({NOT_MODIFIED}, ()) - assert split_token((NOT_MODIFIED, )) == ({NOT_MODIFIED}, ()) + assert split_token((NOT_MODIFIED,)) == ({NOT_MODIFIED}, ()) def test_split_token_inherit(): assert split_token(INHERIT) == ({INHERIT}, ()) - assert split_token((INHERIT, )) == ({INHERIT}, ()) + assert split_token((INHERIT,)) == ({INHERIT}, ()) assert split_token((INHERIT, 'foo', 'bar')) == ({INHERIT}, ('foo', 'bar')) diff --git a/tests/ext/test_ods.py b/tests/ext/test_ods.py index a07ae81..a303a81 100644 --- a/tests/ext/test_ods.py +++ b/tests/ext/test_ods.py @@ -14,16 +14,11 @@ class ResponseMock: return {} else: self.count += 1 - return { - 'records': self.json_value, - } + return {'records': self.json_value} def test_read_from_opendatasoft_api(): extract = OpenDataSoftAPI(dataset='test-a-set') - with patch('requests.get', return_value=ResponseMock([ - {'fields': {'foo': 'bar'}}, - {'fields': {'foo': 'zab'}}, - ])): + with patch('requests.get', return_value=ResponseMock([{'fields': {'foo': 'bar'}}, {'fields': {'foo': 'zab'}}])): for line in extract('http://example.com/', ValueHolder(0)): assert 'foo' in line diff --git a/tests/features/test_inherit.py b/tests/features/test_inherit.py index 92b943b..3d42405 100644 --- a/tests/features/test_inherit.py +++ b/tests/features/test_inherit.py @@ -1,10 +1,7 @@ from bonobo.constants import INHERIT from bonobo.util.testing import BufferingNodeExecutionContext -messages = [ - ('Hello', ), - ('Goodbye', ), -] +messages = [('Hello',), ('Goodbye',)] def append(*args): @@ -15,7 +12,7 @@ def test_inherit(): with BufferingNodeExecutionContext(append) as context: context.write_sync(*messages) - assert context.get_buffer() == list(map(lambda x: x + ('!', ), messages)) + assert context.get_buffer() == list(map(lambda x: x + ('!',), messages)) def test_inherit_bag_tuple(): @@ -24,4 +21,4 @@ def test_inherit_bag_tuple(): context.write_sync(*messages) assert context.get_output_fields() == ('message', '0') - assert context.get_buffer() == list(map(lambda x: x + ('!', ), messages)) + assert context.get_buffer() == list(map(lambda x: x + ('!',), messages)) diff --git a/tests/features/test_not_modified.py b/tests/features/test_not_modified.py index 20f6f96..247765d 100644 --- a/tests/features/test_not_modified.py +++ b/tests/features/test_not_modified.py @@ -7,10 +7,7 @@ def useless(*args, **kwargs): def test_not_modified(): - input_messages = [ - ('foo', 'bar'), - ('foo', 'baz'), - ] + input_messages = [('foo', 'bar'), ('foo', 'baz')] with BufferingNodeExecutionContext(useless) as context: context.write_sync(*input_messages) diff --git a/tests/nodes/io/test_csv.py b/tests/nodes/io/test_csv.py index f862ca7..f347866 100644 --- a/tests/nodes/io/test_csv.py +++ b/tests/nodes/io/test_csv.py @@ -6,8 +6,9 @@ import pytest from bonobo import CsvReader, CsvWriter from bonobo.constants import EMPTY -from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext, WriterTest, ConfigurableNodeTest, \ - ReaderTest +from bonobo.util.testing import ( + BufferingNodeExecutionContext, ConfigurableNodeTest, FilesystemTester, ReaderTest, WriterTest +) csv_tester = FilesystemTester('csv') csv_tester.input_data = 'a,b,c\na foo,b foo,c foo\na bar,b bar,c bar' @@ -23,15 +24,10 @@ def test_read_csv_from_file_kwargs(tmpdir): with BufferingNodeExecutionContext(CsvReader(filename, **defaults), services=services) as context: context.write_sync(EMPTY) - assert context.get_buffer_args_as_dicts() == [{ - 'a': 'a foo', - 'b': 'b foo', - 'c': 'c foo', - }, { - 'a': 'a bar', - 'b': 'b bar', - 'c': 'c bar', - }] + assert context.get_buffer_args_as_dicts() == [ + {'a': 'a foo', 'b': 'b foo', 'c': 'c foo'}, + {'a': 'a bar', 'b': 'b bar', 'c': 'c bar'}, + ] ### @@ -50,22 +46,11 @@ LL = ('i', 'have', 'more', 'values') class CsvReaderTest(Csv, ReaderTest, TestCase): - input_data = '\n'.join(( - 'id,name', - '1,John Doe', - '2,Jane Doe', - ',DPR', - '42,Elon Musk', - )) + input_data = '\n'.join(('id,name', '1,John Doe', '2,Jane Doe', ',DPR', '42,Elon Musk')) def check_output(self, context, *, prepend=None): out = context.get_buffer() - assert out == (prepend or list()) + [ - ('1', 'John Doe'), - ('2', 'Jane Doe'), - ('', 'DPR'), - ('42', 'Elon Musk'), - ] + assert out == (prepend or list()) + [('1', 'John Doe'), ('2', 'Jane Doe'), ('', 'DPR'), ('42', 'Elon Musk')] @incontext() def test_nofields(self, context): @@ -80,12 +65,7 @@ class CsvReaderTest(Csv, ReaderTest, TestCase): context.stop() self.check_output(context, prepend=[('id', 'name')]) - @incontext( - output_fields=( - 'x', - 'y', - ), skip=1 - ) + @incontext(output_fields=('x', 'y'), skip=1) def test_output_fields(self, context): context.write_sync(EMPTY) context.stop() @@ -107,11 +87,7 @@ class CsvWriterTest(Csv, WriterTest, TestCase): context.write_sync(('a', 'b'), ('c', 'd')) context.stop() - assert self.readlines() == ( - 'foo,bar', - 'a,b', - 'c,d', - ) + assert self.readlines() == ('foo,bar', 'a,b', 'c,d') @incontext() def test_fields_from_type(self, context): @@ -127,30 +103,21 @@ class CsvWriterTest(Csv, WriterTest, TestCase): context.write_sync((L1, L2), (L3, L4)) context.stop() - assert self.readlines() == ( - 'a,hey', - 'b,bee', - 'c,see', - 'd,dee', - ) + assert self.readlines() == ('a,hey', 'b,bee', 'c,see', 'd,dee') @incontext() def test_nofields_multiple_args_length_mismatch(self, context): # if length of input vary, then we get a TypeError (unrecoverable) with pytest.raises(TypeError): - context.write_sync((L1, L2), (L3, )) + context.write_sync((L1, L2), (L3,)) @incontext() def test_nofields_single_arg(self, context): # single args are just dumped, shapes can vary. - context.write_sync((L1, ), (LL, ), (L3, )) + context.write_sync((L1,), (LL,), (L3,)) context.stop() - assert self.readlines() == ( - 'a,hey', - 'i,have,more,values', - 'c,see', - ) + assert self.readlines() == ('a,hey', 'i,have,more,values', 'c,see') @incontext() def test_nofields_empty_args(self, context): diff --git a/tests/nodes/io/test_file.py b/tests/nodes/io/test_file.py index ee8548b..1ad443f 100644 --- a/tests/nodes/io/test_file.py +++ b/tests/nodes/io/test_file.py @@ -21,10 +21,7 @@ def test_file_writer_contextless(tmpdir): @pytest.mark.parametrize( 'lines,output', - [ - (('ACME', ), 'ACME'), # one line... - (('Foo', 'Bar', 'Baz'), 'Foo\nBar\nBaz'), # more than one line... - ] + [(('ACME',), 'ACME'), (('Foo', 'Bar', 'Baz'), 'Foo\nBar\nBaz')], # one line... # more than one line... ) def test_file_writer_in_context(tmpdir, lines, output): fs, filename, services = txt_tester.get_services_for_writer(tmpdir) @@ -44,5 +41,5 @@ def test_file_reader(tmpdir): output = context.get_buffer() assert len(output) == 2 - assert output[0] == ('Hello', ) - assert output[1] == ('World', ) + assert output[0] == ('Hello',) + assert output[1] == ('World',) diff --git a/tests/nodes/io/test_json.py b/tests/nodes/io/test_json.py index c2caedd..575a7e8 100644 --- a/tests/nodes/io/test_json.py +++ b/tests/nodes/io/test_json.py @@ -4,10 +4,9 @@ from unittest import TestCase import pytest -from bonobo import JsonReader, JsonWriter -from bonobo import LdjsonReader, LdjsonWriter +from bonobo import JsonReader, JsonWriter, LdjsonReader, LdjsonWriter from bonobo.constants import EMPTY -from bonobo.util.testing import WriterTest, ReaderTest, ConfigurableNodeTest +from bonobo.util.testing import ConfigurableNodeTest, ReaderTest, WriterTest FOOBAR = {'foo': 'bar'} OD_ABC = OrderedDict((('a', 'A'), ('b', 'B'), ('c', 'C'))) @@ -34,14 +33,7 @@ class JsonReaderDictsTest(Json, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ({ - "foo": "bar" - }, ), - ({ - "baz": "boz" - }, ), - ] + assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)] class JsonReaderListsTest(Json, ReaderTest, TestCase): @@ -52,20 +44,14 @@ class JsonReaderListsTest(Json, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] class JsonReaderStringsTest(Json, ReaderTest, TestCase): @@ -76,22 +62,14 @@ class JsonReaderStringsTest(Json, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [('foo',), ('bar',), ('baz',)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [('foo',), ('bar',), ('baz',)] class JsonWriterTest(Json, WriterTest, TestCase): @@ -101,10 +79,7 @@ class JsonWriterTest(Json, WriterTest, TestCase): context.write_sync(('a', 'b'), ('c', 'd')) context.stop() - assert self.readlines() == ( - '[{"foo": "a", "bar": "b"},', - '{"foo": "c", "bar": "d"}]', - ) + assert self.readlines() == ('[{"foo": "a", "bar": "b"},', '{"foo": "c", "bar": "d"}]') @incontext() def test_fields_from_type(self, context): @@ -112,10 +87,7 @@ class JsonWriterTest(Json, WriterTest, TestCase): context.write_sync((1, 2), (3, 4)) context.stop() - assert self.readlines() == ( - '[{"x": 1, "y": 2},', - '{"x": 3, "y": 4}]', - ) + assert self.readlines() == ('[{"x": 1, "y": 2},', '{"x": 3, "y": 4}]') @incontext() def test_nofields_multiple_args(self, context): @@ -144,11 +116,7 @@ class JsonWriterTest(Json, WriterTest, TestCase): context.write_sync(FOOBAR, OD_ABC, FOOBAZ) context.stop() - assert self.readlines() == ( - '[{"foo": "bar"},', - '{"a": "A", "b": "B", "c": "C"},', - '{"foo": "baz"}]', - ) + assert self.readlines() == ('[{"foo": "bar"},', '{"a": "A", "b": "B", "c": "C"},', '{"foo": "baz"}]') @incontext() def test_nofields_empty_args(self, context): @@ -156,7 +124,7 @@ class JsonWriterTest(Json, WriterTest, TestCase): context.write_sync(EMPTY, EMPTY, EMPTY) context.stop() - assert self.readlines() == ('[]', ) + assert self.readlines() == ('[]',) ### @@ -178,14 +146,7 @@ class LdjsonReaderDictsTest(Ldjson, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ({ - "foo": "bar" - }, ), - ({ - "baz": "boz" - }, ), - ] + assert context.get_buffer() == [({"foo": "bar"},), ({"baz": "boz"},)] class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase): @@ -196,20 +157,14 @@ class LdjsonReaderListsTest(Ldjson, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ([1, 2, 3], ), - ([4, 5, 6], ), - ] + assert context.get_buffer() == [([1, 2, 3],), ([4, 5, 6],)] class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase): @@ -220,22 +175,14 @@ class LdjsonReaderStringsTest(Ldjson, ReaderTest, TestCase): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [('foo',), ('bar',), ('baz',)] @incontext(output_type=tuple) def test_output_type(self, context): context.write_sync(EMPTY) context.stop() - assert context.get_buffer() == [ - ('foo', ), - ('bar', ), - ('baz', ), - ] + assert context.get_buffer() == [('foo',), ('bar',), ('baz',)] class LdjsonWriterTest(Ldjson, WriterTest, TestCase): @@ -253,10 +200,7 @@ class LdjsonWriterTest(Ldjson, WriterTest, TestCase): context.write_sync((1, 2), (3, 4)) context.stop() - assert self.readlines() == ( - '{"x": 1, "y": 2}', - '{"x": 3, "y": 4}', - ) + assert self.readlines() == ('{"x": 1, "y": 2}', '{"x": 3, "y": 4}') @incontext() def test_nofields_multiple_args(self, context): @@ -285,11 +229,7 @@ class LdjsonWriterTest(Ldjson, WriterTest, TestCase): context.write_sync(FOOBAR, OD_ABC, FOOBAZ) context.stop() - assert self.readlines() == ( - '{"foo": "bar"}', - '{"a": "A", "b": "B", "c": "C"}', - '{"foo": "baz"}', - ) + assert self.readlines() == ('{"foo": "bar"}', '{"a": "A", "b": "B", "c": "C"}', '{"foo": "baz"}') @incontext() def test_nofields_empty_args(self, context): diff --git a/tests/nodes/io/test_pickle.py b/tests/nodes/io/test_pickle.py index 0662848..cc67c91 100644 --- a/tests/nodes/io/test_pickle.py +++ b/tests/nodes/io/test_pickle.py @@ -32,7 +32,4 @@ def test_read_pickled_list_from_file(tmpdir): output = context.get_buffer() assert context.get_output_fields() == ('a', 'b', 'c') - assert output == [ - ('a foo', 'b foo', 'c foo'), - ('a bar', 'b bar', 'c bar'), - ] + assert output == [('a foo', 'b foo', 'c foo'), ('a bar', 'b bar', 'c bar')] diff --git a/tests/nodes/test_basics.py b/tests/nodes/test_basics.py index 4b6a8be..90c0f20 100644 --- a/tests/nodes/test_basics.py +++ b/tests/nodes/test_basics.py @@ -5,9 +5,9 @@ from unittest.mock import MagicMock import pytest import bonobo -from bonobo.constants import NOT_MODIFIED, EMPTY -from bonobo.util import ensure_tuple, ValueHolder -from bonobo.util.testing import BufferingNodeExecutionContext, StaticNodeTest, ConfigurableNodeTest +from bonobo.constants import EMPTY, NOT_MODIFIED +from bonobo.util import ValueHolder, ensure_tuple +from bonobo.util.testing import BufferingNodeExecutionContext, ConfigurableNodeTest, StaticNodeTest class CountTest(StaticNodeTest, TestCase): @@ -26,7 +26,7 @@ class CountTest(StaticNodeTest, TestCase): def test_execution(self): with self.execute() as context: context.write_sync(*([EMPTY] * 42)) - assert context.get_buffer() == [(42, )] + assert context.get_buffer() == [(42,)] class IdentityTest(StaticNodeTest, TestCase): @@ -98,14 +98,11 @@ def test_fixedwindow(): with BufferingNodeExecutionContext(bonobo.FixedWindow(2)) as context: context.write_sync(*range(9)) - assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), ( - 8, - None, - )] + assert context.get_buffer() == [(0, 1), (2, 3), (4, 5), (6, 7), (8, None)] with BufferingNodeExecutionContext(bonobo.FixedWindow(1)) as context: context.write_sync(*range(3)) - assert context.get_buffer() == [(0, ), (1, ), (2, )] + assert context.get_buffer() == [(0,), (1,), (2,)] def test_methodcaller(): diff --git a/tests/plugins/test_console.py b/tests/plugins/test_console.py index 543d341..3a7eb3b 100644 --- a/tests/plugins/test_console.py +++ b/tests/plugins/test_console.py @@ -1,10 +1,11 @@ from unittest.mock import MagicMock +from whistle import EventDispatcher + import bonobo from bonobo.execution import events from bonobo.execution.contexts.graph import GraphExecutionContext from bonobo.plugins.console import ConsoleOutputPlugin -from whistle import EventDispatcher def test_register_unregister(): diff --git a/tests/structs/test_graphs.py b/tests/structs/test_graphs.py index 7f3a58d..68115cf 100644 --- a/tests/structs/test_graphs.py +++ b/tests/structs/test_graphs.py @@ -1,7 +1,7 @@ -import pytest - from unittest.mock import sentinel +import pytest + from bonobo.constants import BEGIN from bonobo.structs import Graph @@ -48,24 +48,14 @@ def test_graph_add_chain(): def test_graph_topological_sort(): g = Graph() - g.add_chain( - sentinel.a1, - sentinel.a2, - sentinel.a3, - _input=None, - _output=None, - ) + g.add_chain(sentinel.a1, sentinel.a2, sentinel.a3, _input=None, _output=None) assert g.topologically_sorted_indexes == (0, 1, 2) assert g[0] == sentinel.a1 assert g[1] == sentinel.a2 assert g[2] == sentinel.a3 - g.add_chain( - sentinel.b1, - sentinel.b2, - _output=sentinel.a2, - ) + g.add_chain(sentinel.b1, sentinel.b2, _output=sentinel.a2) assert g.topologically_sorted_indexes[-2:] == (1, 2) assert g.topologically_sorted_indexes.index(3) < g.topologically_sorted_indexes.index(4) diff --git a/tests/structs/test_inputs.py b/tests/structs/test_inputs.py index d2ce827..dc6af95 100644 --- a/tests/structs/test_inputs.py +++ b/tests/structs/test_inputs.py @@ -19,7 +19,7 @@ from queue import Empty import pytest from bonobo.constants import BEGIN, END -from bonobo.errors import InactiveWritableError, InactiveReadableError +from bonobo.errors import InactiveReadableError, InactiveWritableError from bonobo.structs.inputs import Input diff --git a/tests/test_execution.py b/tests/test_execution.py index 92cd30c..5702cd1 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -10,7 +10,7 @@ def generate_integers(): def square(i): - return i**2 + return i ** 2 def results(f, context): diff --git a/tests/util/test_bags.py b/tests/util/test_bags.py index 469fbb6..6e26b9d 100644 --- a/tests/util/test_bags.py +++ b/tests/util/test_bags.py @@ -147,14 +147,14 @@ class TestBagType(unittest.TestCase): self.assertEqual(Zero()._asdict(), {}) self.assertEqual(Zero()._fields, ()) - Dot = BagType('Dot', ('d', )) - self.assertEqual(Dot(1), (1, )) - self.assertEqual(Dot._make([1]), (1, )) + Dot = BagType('Dot', ('d',)) + self.assertEqual(Dot(1), (1,)) + self.assertEqual(Dot._make([1]), (1,)) self.assertEqual(Dot(1).d, 1) self.assertEqual(repr(Dot(1)), 'Dot(d=1)') self.assertEqual(Dot(1)._asdict(), {'d': 1}) - self.assertEqual(Dot(1)._replace(d=999), (999, )) - self.assertEqual(Dot(1)._fields, ('d', )) + self.assertEqual(Dot(1)._replace(d=999), (999,)) + self.assertEqual(Dot(1)._fields, ('d',)) n = 5000 if sys.version_info >= (3, 7) else 254 names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) @@ -178,7 +178,7 @@ class TestBagType(unittest.TestCase): def test_pickle(self): p = TBag(x=10, y=20, z=30) - for module in (pickle, ): + for module in (pickle,): loads = getattr(module, 'loads') dumps = getattr(module, 'dumps') for protocol in range(-1, module.HIGHEST_PROTOCOL + 1): @@ -206,25 +206,191 @@ class TestBagType(unittest.TestCase): # Broader test of all interesting names taken from the code, old # template, and an example words = { - 'Alias', 'At', 'AttributeError', 'Build', 'Bypass', 'Create', 'Encountered', 'Expected', 'Field', 'For', - 'Got', 'Helper', 'IronPython', 'Jython', 'KeyError', 'Make', 'Modify', 'Note', 'OrderedDict', 'Point', - 'Return', 'Returns', 'Type', 'TypeError', 'Used', 'Validate', 'ValueError', 'Variables', 'a', 'accessible', - 'add', 'added', 'all', 'also', 'an', 'arg_list', 'args', 'arguments', 'automatically', 'be', 'build', - 'builtins', 'but', 'by', 'cannot', 'class_namespace', 'classmethod', 'cls', 'collections', 'convert', - 'copy', 'created', 'creation', 'd', 'debugging', 'defined', 'dict', 'dictionary', 'doc', 'docstring', - 'docstrings', 'duplicate', 'effect', 'either', 'enumerate', 'environments', 'error', 'example', 'exec', 'f', - 'f_globals', 'field', 'field_names', 'fields', 'formatted', 'frame', 'function', 'functions', 'generate', - 'getter', 'got', 'greater', 'has', 'help', 'identifiers', 'indexable', 'instance', 'instantiate', - 'interning', 'introspection', 'isidentifier', 'isinstance', 'itemgetter', 'iterable', 'join', 'keyword', - 'keywords', 'kwds', 'len', 'like', 'list', 'map', 'maps', 'message', 'metadata', 'method', 'methods', - 'module', 'module_name', 'must', 'name', 'named', 'namedtuple', 'namedtuple_', 'names', 'namespace', - 'needs', 'new', 'nicely', 'num_fields', 'number', 'object', 'of', 'operator', 'option', 'p', 'particular', - 'pickle', 'pickling', 'plain', 'pop', 'positional', 'property', 'r', 'regular', 'rename', 'replace', - 'replacing', 'repr', 'repr_fmt', 'representation', 'result', 'reuse_itemgetter', 's', 'seen', 'sequence', - 'set', 'side', 'specified', 'split', 'start', 'startswith', 'step', 'str', 'string', 'strings', 'subclass', - 'sys', 'targets', 'than', 'the', 'their', 'this', 'to', 'tuple_new', 'type', 'typename', 'underscore', - 'unexpected', 'unpack', 'up', 'use', 'used', 'user', 'valid', 'values', 'variable', 'verbose', 'where', - 'which', 'work', 'x', 'y', 'z', 'zip' + 'Alias', + 'At', + 'AttributeError', + 'Build', + 'Bypass', + 'Create', + 'Encountered', + 'Expected', + 'Field', + 'For', + 'Got', + 'Helper', + 'IronPython', + 'Jython', + 'KeyError', + 'Make', + 'Modify', + 'Note', + 'OrderedDict', + 'Point', + 'Return', + 'Returns', + 'Type', + 'TypeError', + 'Used', + 'Validate', + 'ValueError', + 'Variables', + 'a', + 'accessible', + 'add', + 'added', + 'all', + 'also', + 'an', + 'arg_list', + 'args', + 'arguments', + 'automatically', + 'be', + 'build', + 'builtins', + 'but', + 'by', + 'cannot', + 'class_namespace', + 'classmethod', + 'cls', + 'collections', + 'convert', + 'copy', + 'created', + 'creation', + 'd', + 'debugging', + 'defined', + 'dict', + 'dictionary', + 'doc', + 'docstring', + 'docstrings', + 'duplicate', + 'effect', + 'either', + 'enumerate', + 'environments', + 'error', + 'example', + 'exec', + 'f', + 'f_globals', + 'field', + 'field_names', + 'fields', + 'formatted', + 'frame', + 'function', + 'functions', + 'generate', + 'getter', + 'got', + 'greater', + 'has', + 'help', + 'identifiers', + 'indexable', + 'instance', + 'instantiate', + 'interning', + 'introspection', + 'isidentifier', + 'isinstance', + 'itemgetter', + 'iterable', + 'join', + 'keyword', + 'keywords', + 'kwds', + 'len', + 'like', + 'list', + 'map', + 'maps', + 'message', + 'metadata', + 'method', + 'methods', + 'module', + 'module_name', + 'must', + 'name', + 'named', + 'namedtuple', + 'namedtuple_', + 'names', + 'namespace', + 'needs', + 'new', + 'nicely', + 'num_fields', + 'number', + 'object', + 'of', + 'operator', + 'option', + 'p', + 'particular', + 'pickle', + 'pickling', + 'plain', + 'pop', + 'positional', + 'property', + 'r', + 'regular', + 'rename', + 'replace', + 'replacing', + 'repr', + 'repr_fmt', + 'representation', + 'result', + 'reuse_itemgetter', + 's', + 'seen', + 'sequence', + 'set', + 'side', + 'specified', + 'split', + 'start', + 'startswith', + 'step', + 'str', + 'string', + 'strings', + 'subclass', + 'sys', + 'targets', + 'than', + 'the', + 'their', + 'this', + 'to', + 'tuple_new', + 'type', + 'typename', + 'underscore', + 'unexpected', + 'unpack', + 'up', + 'use', + 'used', + 'user', + 'valid', + 'values', + 'variable', + 'verbose', + 'where', + 'which', + 'work', + 'x', + 'y', + 'z', + 'zip', } sorted_words = tuple(sorted(words)) T = BagType('T', sorted_words) @@ -252,7 +418,7 @@ class TestBagType(unittest.TestCase): self.assertEqual(t.__getnewargs__(), values) def test_repr(self): - A = BagType('A', ('x', )) + A = BagType('A', ('x',)) self.assertEqual(repr(A(1)), 'A(x=1)') # repr should show the name of the subclass @@ -273,6 +439,18 @@ class TestBagType(unittest.TestCase): def test_annoying_attribute_names(self): self._create( - '__slots__', '__getattr__', '_attrs', '_fields', '__new__', '__getnewargs__', '__repr__', '_make', 'get', - '_replace', '_asdict', '_cls', 'self', 'tuple' + '__slots__', + '__getattr__', + '_attrs', + '_fields', + '__new__', + '__getnewargs__', + '__repr__', + '_make', + 'get', + '_replace', + '_asdict', + '_cls', + 'self', + 'tuple', ) diff --git a/tests/util/test_collections.py b/tests/util/test_collections.py index bdc6ec1..bf7b097 100644 --- a/tests/util/test_collections.py +++ b/tests/util/test_collections.py @@ -1,7 +1,7 @@ import pytest -from bonobo.util import sortedlist, ensure_tuple -from bonobo.util.collections import tuplize, cast +from bonobo.util import ensure_tuple, sortedlist +from bonobo.util.collections import cast, tuplize def test_sortedlist(): @@ -14,8 +14,8 @@ def test_sortedlist(): def test_ensure_tuple(): - assert ensure_tuple('a') == ('a', ) - assert ensure_tuple(('a', )) == ('a', ) + assert ensure_tuple('a') == ('a',) + assert ensure_tuple(('a',)) == ('a',) assert ensure_tuple(()) is () diff --git a/tests/util/test_objects.py b/tests/util/test_objects.py index 9a3696e..1cffadf 100644 --- a/tests/util/test_objects.py +++ b/tests/util/test_objects.py @@ -2,7 +2,7 @@ import operator import pytest -from bonobo.util.objects import Wrapper, get_name, ValueHolder, get_attribute_or_create +from bonobo.util.objects import ValueHolder, Wrapper, get_attribute_or_create, get_name from bonobo.util.testing import optional_contextmanager @@ -65,10 +65,7 @@ def test_valueholder_notequal(): assert not (x != 42) -@pytest.mark.parametrize('rlo,rhi', [ - (1, 2), - ('a', 'b'), -]) +@pytest.mark.parametrize('rlo,rhi', [(1, 2), ('a', 'b')]) def test_valueholder_ordering(rlo, rhi): vlo, vhi = ValueHolder(rlo), ValueHolder(rhi) @@ -129,15 +126,27 @@ def test_get_attribute_or_create(): unsupported_operations = { int: {operator.matmul}, str: { - operator.sub, operator.mul, operator.matmul, operator.floordiv, operator.truediv, operator.mod, divmod, - operator.pow, operator.lshift, operator.rshift, operator.and_, operator.xor, operator.or_ + operator.sub, + operator.mul, + operator.matmul, + operator.floordiv, + operator.truediv, + operator.mod, + divmod, + operator.pow, + operator.lshift, + operator.rshift, + operator.and_, + operator.xor, + operator.or_, }, } @pytest.mark.parametrize('x,y', [(5, 3), (0, 10), (0, 0), (1, 1), ('foo', 'bar'), ('', 'baz!')]) @pytest.mark.parametrize( - 'operation,inplace_operation', [ + 'operation,inplace_operation', + [ (operator.add, operator.iadd), (operator.sub, operator.isub), (operator.mul, operator.imul), @@ -152,7 +161,7 @@ unsupported_operations = { (operator.and_, operator.iand), (operator.xor, operator.ixor), (operator.or_, operator.ior), - ] + ], ) def test_valueholder_integer_operations(x, y, operation, inplace_operation): v = ValueHolder(x) diff --git a/tests/util/test_resolvers.py b/tests/util/test_resolvers.py index 0de3003..1ef22de 100644 --- a/tests/util/test_resolvers.py +++ b/tests/util/test_resolvers.py @@ -15,4 +15,4 @@ def test_resolve_options(): def test_resolve_transformations(): - assert _resolve_transformations(('PrettyPrinter', )) == (bonobo.PrettyPrinter, ) + assert _resolve_transformations(('PrettyPrinter',)) == (bonobo.PrettyPrinter,) diff --git a/tests/util/test_statistics.py b/tests/util/test_statistics.py index 9eae0c0..41bccd3 100644 --- a/tests/util/test_statistics.py +++ b/tests/util/test_statistics.py @@ -3,10 +3,7 @@ from bonobo.util.statistics import WithStatistics class MyThingWithStats(WithStatistics): def get_statistics(self, *args, **kwargs): - return ( - ('foo', 42), - ('bar', 69), - ) + return (('foo', 42), ('bar', 69)) def test_with_statistics(): From 408e7523d44367f0e207373da8978ba779324c48 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:37:00 +0200 Subject: [PATCH 09/35] style: disable codacy on tests, as complaining on assert usage is plain stupidity here. --- .codacy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codacy.yml b/.codacy.yml index 390e58f..316e0be 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -2,3 +2,4 @@ exclude_paths: - bonobo/examples/ - bonobo/ext/ + - tests/ From 3f14082e50093f392dc4819b3f6480cf4b0a0dff Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:42:15 +0200 Subject: [PATCH 10/35] style: fix codacy config. --- .codacy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.codacy.yml b/.codacy.yml index 316e0be..5ca8fee 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,5 +1,5 @@ --- exclude_paths: - - bonobo/examples/ - - bonobo/ext/ - - tests/ + - bonobo/examples/** + - bonobo/ext/** + - tests/** From 97618099937abfe8ce49f774f3d083a2d7a17a68 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:49:15 +0200 Subject: [PATCH 11/35] style: removes codacy linter from benchmark dir. --- .codacy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codacy.yml b/.codacy.yml index 5ca8fee..cacb366 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,5 +1,6 @@ --- exclude_paths: + - benchmarks/** - bonobo/examples/** - bonobo/ext/** - tests/** From 7269f2cc546cf1aa905a62840c357748455c0dfc Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:53:41 +0200 Subject: [PATCH 12/35] smell: removes unused import, as it is imported by get_versions below. --- bonobo/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bonobo/__init__.py b/bonobo/__init__.py index 97d5443..2ee43ba 100644 --- a/bonobo/__init__.py +++ b/bonobo/__init__.py @@ -23,7 +23,6 @@ __version__ = __version__ def _repr_html_(): """This allows to easily display a version snippet in Jupyter.""" - from bonobo.util.pkgs import bonobo_packages from bonobo.commands.version import get_versions return ( From 0b32b9751ca928a118a2cf2648d58fb3c1b5e03b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:56:42 +0200 Subject: [PATCH 13/35] smell: rename unused var --- bonobo/commands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/commands/init.py b/bonobo/commands/init.py index 6d4b217..aa4d841 100644 --- a/bonobo/commands/init.py +++ b/bonobo/commands/init.py @@ -33,7 +33,7 @@ class InitCommand(BaseCommand): self.logger.info('Generated {} using template {!r}.'.format(filename, template_name)) def create_package(self, *, filename): - name, ext = os.path.splitext(filename) + _, ext = os.path.splitext(filename) if ext != '': raise ValueError('Package names should not have an extension.') From f0a18229b82e752b52b42a128f22799af5e9b830 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:56:51 +0200 Subject: [PATCH 14/35] smell: rename unused imports --- bonobo/config/services.py | 2 -- bonobo/nodes/basics.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/bonobo/config/services.py b/bonobo/config/services.py index 5a9f392..8c0abb1 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -1,5 +1,3 @@ -import inspect -import pprint import re import threading import types diff --git a/bonobo/nodes/basics.py b/bonobo/nodes/basics.py index 665eedc..4f7fd8d 100644 --- a/bonobo/nodes/basics.py +++ b/bonobo/nodes/basics.py @@ -58,8 +58,6 @@ class Limit(Configurable): def Tee(f): - from bonobo.constants import NOT_MODIFIED - @functools.wraps(f) def wrapped(*args, **kwargs): nonlocal f From b9768fac6abfa34ca301e20188ec9c70f7a46450 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:57:20 +0200 Subject: [PATCH 15/35] smell: rename unused imports --- bonobo/nodes/io/csv.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bonobo/nodes/io/csv.py b/bonobo/nodes/io/csv.py index c6fbdb6..8f3fb32 100644 --- a/bonobo/nodes/io/csv.py +++ b/bonobo/nodes/io/csv.py @@ -1,12 +1,11 @@ import csv -from bonobo.config import Option, use_context, use_raw_input +from bonobo.config import Option, use_context from bonobo.config.options import Method, RenamedOption from bonobo.constants import NOT_MODIFIED from bonobo.nodes.io.base import FileHandler from bonobo.nodes.io.file import FileReader, FileWriter from bonobo.util import ensure_tuple -from bonobo.util.bags import BagType class CsvHandler(FileHandler): From ba60f799e2dd1396890b6f8ce6006f38af364deb Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:57:48 +0200 Subject: [PATCH 16/35] smell: rename unused import --- bonobo/util/environ.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bonobo/util/environ.py b/bonobo/util/environ.py index 6cdb76f..2589959 100644 --- a/bonobo/util/environ.py +++ b/bonobo/util/environ.py @@ -57,8 +57,6 @@ def get_argument_parser(parser=None): :return: """ if parser is None: - import argparse - parser = argparse.ArgumentParser() # Store globally to be able to warn the user about the fact he's probably wrong not to pass a parser to From a3120e541503b744a57f8bbca9e15899d6966dcf Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 06:58:47 +0200 Subject: [PATCH 17/35] style: dont lint generated js --- .codacy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codacy.yml b/.codacy.yml index cacb366..2646b04 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,6 +1,7 @@ --- exclude_paths: - benchmarks/** + - bonobo/contrib/jupyter/**.js - bonobo/examples/** - bonobo/ext/** - tests/** From 23e93c69700ff25d45d2f8c72cae36918a58e66e Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:00:53 +0200 Subject: [PATCH 18/35] style: dont lint bin helpers --- .codacy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codacy.yml b/.codacy.yml index 2646b04..0044a02 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,5 +1,6 @@ --- exclude_paths: + - bin/** - benchmarks/** - bonobo/contrib/jupyter/**.js - bonobo/examples/** From 7ae1fa82f77b987f63671c566196378432765230 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:06:12 +0200 Subject: [PATCH 19/35] smell: removes global catches, even if it may help in such critical places. --- bonobo/config/configurables.py | 2 +- bonobo/config/processors.py | 2 +- bonobo/execution/contexts/node.py | 2 +- bonobo/execution/strategies/executor.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bonobo/config/configurables.py b/bonobo/config/configurables.py index a894a84..758830d 100644 --- a/bonobo/config/configurables.py +++ b/bonobo/config/configurables.py @@ -67,7 +67,7 @@ class ConfigurableMeta(type): try: import _functools -except: +except Exception: import functools PartiallyConfigured = functools.partial diff --git a/bonobo/config/processors.py b/bonobo/config/processors.py index 7083484..6f9400b 100644 --- a/bonobo/config/processors.py +++ b/bonobo/config/processors.py @@ -133,7 +133,7 @@ class ContextCurrifier: try: # todo yield from ? how to ? processor.send(self._stack_values.pop()) - except StopIteration as exc: + except StopIteration: # This is normal, and wanted. pass else: diff --git a/bonobo/execution/contexts/node.py b/bonobo/execution/contexts/node.py index 1bc84c4..dfb2a1c 100644 --- a/bonobo/execution/contexts/node.py +++ b/bonobo/execution/contexts/node.py @@ -160,7 +160,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): if self._stack: try: self._stack.teardown() - except: + except Exception: self.fatal(sys.exc_info()) super().stop() diff --git a/bonobo/execution/strategies/executor.py b/bonobo/execution/strategies/executor.py index 8f33574..85de43b 100644 --- a/bonobo/execution/strategies/executor.py +++ b/bonobo/execution/strategies/executor.py @@ -29,7 +29,7 @@ class ExecutorStrategy(Strategy): with self.create_executor(graph) as executor: try: context.start(self.get_starter(executor, futures)) - except: + except Exception: logger.critical('Exception caught while starting execution context.', exc_info=sys.exc_info()) while context.alive: @@ -53,14 +53,14 @@ class ExecutorStrategy(Strategy): try: with node: node.loop() - except: + except Exception: logging.getLogger(__name__).critical( 'Critical error in threadpool node starter.', exc_info=sys.exc_info() ) try: futures.append(executor.submit(_runner)) - except: + except Exception: logging.getLogger(__name__).critical('futures.append', exc_info=sys.exc_info()) return starter From 349f999d00dd5c58ca9dfffd35ed48e2c2b92ad7 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:08:35 +0200 Subject: [PATCH 20/35] smell: move type checks to use isinstance. --- bonobo/settings.py | 2 +- bonobo/util/bags.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bonobo/settings.py b/bonobo/settings.py index 9c1cfb8..776527e 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -7,7 +7,7 @@ from bonobo.errors import ValidationError def to_bool(s): if s is None: return False - if type(s) is bool: + if isinstance(s, bool): return s if len(s): if s.lower() in ('f', 'false', 'n', 'no', '0'): diff --git a/bonobo/util/bags.py b/bonobo/util/bags.py index 49a7035..2cf0f93 100644 --- a/bonobo/util/bags.py +++ b/bonobo/util/bags.py @@ -125,13 +125,13 @@ def BagType(typename, fields, *, verbose=False, module=None): # message or automatically replace the field name with a valid name. attrs = tuple(map(_uniquify(_make_valid_attr_name), fields)) - if type(fields) is str: + if isinstance(fields, str): raise TypeError('BagType does not support providing fields as a string.') fields = list(map(str, fields)) typename = str(typename) for i, name in enumerate([typename] + fields): - if type(name) is not str: + if not isinstance(name, str): raise TypeError('Type names and field names must be strings, got {name!r}'.format(name=name)) if not i: if not name.isidentifier(): From 2a6fdce92c034b24afe3a9e57d8df083dbce6824 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:08:54 +0200 Subject: [PATCH 21/35] style: removes codacy linter from sphinx doc dir. --- .codacy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codacy.yml b/.codacy.yml index 0044a02..60895e9 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -5,4 +5,5 @@ exclude_paths: - bonobo/contrib/jupyter/**.js - bonobo/examples/** - bonobo/ext/** + - docs/** - tests/** From f82923e13bdcafb0007c6e2cef3149053c11761a Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:14:25 +0200 Subject: [PATCH 22/35] smell: removes unnecessary lambda --- bonobo/util/term.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/util/term.py b/bonobo/util/term.py index 2fa02ce..e647ca2 100644 --- a/bonobo/util/term.py +++ b/bonobo/util/term.py @@ -1,2 +1,2 @@ CLEAR_EOL = '\033[0K' -MOVE_CURSOR_UP = lambda n: '\033[{}A'.format(n) +MOVE_CURSOR_UP = '\033[{}A'.format From b9a2c4ce9e14872fd96a5cba4cf3a3006c058556 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:15:05 +0200 Subject: [PATCH 23/35] style: remote setup.py from linter. --- .codacy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.codacy.yml b/.codacy.yml index 60895e9..76823bb 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -1,9 +1,10 @@ --- exclude_paths: - - bin/** - benchmarks/** + - bin/** - bonobo/contrib/jupyter/**.js - bonobo/examples/** - bonobo/ext/** - docs/** + - setup.py - tests/** From adcb210c6403308e49324f8a0f5227c00f212ac5 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:16:12 +0200 Subject: [PATCH 24/35] smell: move commands arg to nonlocal var --- bonobo/commands/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bonobo/commands/__init__.py b/bonobo/commands/__init__.py index fb6de85..7017a5b 100644 --- a/bonobo/commands/__init__.py +++ b/bonobo/commands/__init__.py @@ -26,7 +26,9 @@ def entrypoint(args=None): commands = {} - def register_extension(ext, commands=commands): + def register_extension(ext): + nonlocal commands + try: parser = subparsers.add_parser(ext.name) if isinstance(ext.plugin, type) and issubclass(ext.plugin, BaseCommand): From df84d917c31906ce034e76a660ed99e0436bf3a5 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:25:43 +0200 Subject: [PATCH 25/35] smell: removes whitespace --- bonobo/config/services.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bonobo/config/services.py b/bonobo/config/services.py index 8c0abb1..fa6879d 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -21,32 +21,32 @@ class Service(Option): identifier. For example, you can create a Configurable that has a "database" Service in its attribute, meaning that you'll define which database to use, by name, when creating the instance of this class, then provide an implementation when running the graph using a strategy. - + Example:: - + import bonobo - + class QueryExtractor(bonobo.Configurable): database = bonobo.Service(default='sqlalchemy.engine.default') - + graph = bonobo.Graph( QueryExtractor(database='sqlalchemy.engine.secondary'), *more_transformations, ) - + if __name__ == '__main__': engine = create_engine('... dsn ...') bonobo.run(graph, services={ 'sqlalchemy.engine.secondary': engine }) - + The main goal is not to tie transformations to actual dependencies, so the same can be run in different contexts (stages like preprod, prod, or tenants like client1, client2, or anything you want). .. attribute:: name Service name will be used to retrieve the implementation at runtime. - + """ def __init__(self, name, __doc__=None): From 3ea0952bd3fb9848ef038b622aeec5e20ce9dab1 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:26:14 +0200 Subject: [PATCH 26/35] fix: Setting.set was incorrectly instanciating ValidationError. --- bonobo/settings.py | 4 ++-- tests/test_settings.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bonobo/settings.py b/bonobo/settings.py index 776527e..2141561 100644 --- a/bonobo/settings.py +++ b/bonobo/settings.py @@ -31,7 +31,7 @@ class Setting: def __init__(self, name, default=None, validator=None, formatter=None): self.name = name - if default: + if default is not None: self.default = default if callable(default) else lambda: default else: self.default = lambda: None @@ -51,7 +51,7 @@ class Setting: def set(self, value): value = self.formatter(value) if self.formatter else value if self.validator and not self.validator(value): - raise ValidationError('Invalid value {!r} for setting {}.'.format(value, self.name)) + raise ValidationError(self, 'Invalid value {!r} for setting {}.'.format(value, self.name)) self.value = value def set_if_true(self, value): diff --git a/tests/test_settings.py b/tests/test_settings.py index 5771dc4..9ebed97 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest from bonobo import settings +from bonobo.errors import ValidationError TEST_SETTING = 'TEST_SETTING' @@ -38,6 +39,15 @@ def test_setting(): s.clear() assert s.get() == 'hello' + s = settings.Setting(TEST_SETTING, default=0, validator=lambda x: x == 42) + with pytest.raises(ValidationError): + assert s.get() is 0 + + s.set(42) + + with pytest.raises(ValidationError): + s.set(21) + def test_default_settings(): settings.clear_all() From ba089310208ccd664fc5b50ac66f77e96ba36b33 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:33:11 +0200 Subject: [PATCH 27/35] smell: fixes assert in container constructor. --- bonobo/config/services.py | 7 +++++-- tests/config/test_services.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bonobo/config/services.py b/bonobo/config/services.py index fa6879d..852aae6 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -66,8 +66,11 @@ class Service(Option): class Container(dict): def __new__(cls, *args, **kwargs): if len(args) == 1: - assert not len(kwargs), 'only one usage at a time, my dear.' - if not (args[0]): + if len(kwargs): + raise ValueError( + 'You can either use {} with one positional argument or with keyword arguments, not both.' + ) + if not args[0]: return super().__new__(cls) if isinstance(args[0], cls): return cls diff --git a/tests/config/test_services.py b/tests/config/test_services.py index 5671e90..a4b8c02 100644 --- a/tests/config/test_services.py +++ b/tests/config/test_services.py @@ -130,6 +130,20 @@ def test_requires(): assert svcargs['output'] == vcr.append +def test_constructor(): + c1 = Container(foo='foo', bar='bar') + assert 2 == len(c1) + + c2 = Container({'foo': 'foo', 'bar': 'bar'}) + assert 2 == len(c2) + + assert c1['foo'] == c2['foo'] + assert c1['bar'] == c2['bar'] + + with pytest.raises(ValueError): + Container({'bar': 'bar'}, foo='foo') + + @pytest.mark.parametrize('services', [None, {}]) def test_create_container_empty_values(services): c = create_container(services) From c5aeab37b7d482d36bc5e9b750ab8bea88e0cac0 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:35:39 +0200 Subject: [PATCH 28/35] fix: missing format arg --- bonobo/config/services.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bonobo/config/services.py b/bonobo/config/services.py index 852aae6..207d228 100644 --- a/bonobo/config/services.py +++ b/bonobo/config/services.py @@ -68,7 +68,9 @@ class Container(dict): if len(args) == 1: if len(kwargs): raise ValueError( - 'You can either use {} with one positional argument or with keyword arguments, not both.' + 'You can either use {} with one positional argument or with keyword arguments, not both.'.format( + cls.__name__ + ) ) if not args[0]: return super().__new__(cls) From 1d370a8c2386dc2dc93945eda00ebfe6d450f3a5 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:36:00 +0200 Subject: [PATCH 29/35] smell: switch assert for raise ValueError --- bonobo/contrib/django/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bonobo/contrib/django/commands.py b/bonobo/contrib/django/commands.py index 1fb38c3..a9a2a49 100644 --- a/bonobo/contrib/django/commands.py +++ b/bonobo/contrib/django/commands.py @@ -59,7 +59,8 @@ class ETLCommand(BaseCommand): graph_coll = (graph_coll,) for i, graph in enumerate(graph_coll): - assert isinstance(graph, bonobo.Graph), 'Invalid graph provided.' + if not isinstance(graph, bonobo.Graph): + raise ValueError('Expected a Graph instance, got {!r}.'.format(graph)) print(term.lightwhite('{}. {}'.format(i + 1, graph.name))) result = bonobo.run(graph, services=services, strategy=strategy) results.append(result) From 5e99ef9eb48f7e09cefe3053361a6cd323634423 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:37:29 +0200 Subject: [PATCH 30/35] style: removes bonobo/util/testing.py from linter as it uses asserts for tests. --- .codacy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codacy.yml b/.codacy.yml index 76823bb..0dad881 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -5,6 +5,7 @@ exclude_paths: - bonobo/contrib/jupyter/**.js - bonobo/examples/** - bonobo/ext/** + - bonobo/util/testing.py - docs/** - setup.py - tests/** From 8570f78036532058376eaca824ca7022653bb6e0 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:38:21 +0200 Subject: [PATCH 31/35] smell: removes unused variables. --- bonobo/execution/contexts/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bonobo/execution/contexts/base.py b/bonobo/execution/contexts/base.py index 953f13c..80c6eab 100644 --- a/bonobo/execution/contexts/base.py +++ b/bonobo/execution/contexts/base.py @@ -13,7 +13,7 @@ from bonobo.util.objects import Wrapper, get_name def recoverable(error_handler): try: yield - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except error_handler(*sys.exc_info(), level=ERROR) @@ -21,7 +21,7 @@ def recoverable(error_handler): def unrecoverable(error_handler): try: yield - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except error_handler(*sys.exc_info(), level=ERROR) raise # raise unrecoverableerror from x ? From 3cb616fc6597783284e205b369b951a6a1ec4423 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:41:01 +0200 Subject: [PATCH 32/35] smell: switch type check with isinstance --- bonobo/execution/contexts/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bonobo/execution/contexts/node.py b/bonobo/execution/contexts/node.py index dfb2a1c..42fe811 100644 --- a/bonobo/execution/contexts/node.py +++ b/bonobo/execution/contexts/node.py @@ -177,7 +177,7 @@ class NodeExecutionContext(BaseContext, WithStatistics): if self._input_type is not None: raise RuntimeError('Cannot override input type, already have %r.', self._input_type) - if type(input_type) is not type: + if not isinstance(input_type, type): raise UnrecoverableTypeError('Input types must be regular python types.') if not issubclass(input_type, tuple): From e4cbd579402732581ed186ff2875a8f9c6467cdc Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 07:41:30 +0200 Subject: [PATCH 33/35] smell: removes trailing space --- bonobo/execution/strategies/__init__.py | 2 +- bonobo/nodes/filter.py | 4 ++-- bonobo/plugins/__init__.py | 4 ++-- bonobo/util/objects.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bonobo/execution/strategies/__init__.py b/bonobo/execution/strategies/__init__.py index 90a13de..34cb7a2 100644 --- a/bonobo/execution/strategies/__init__.py +++ b/bonobo/execution/strategies/__init__.py @@ -23,7 +23,7 @@ DEFAULT_STRATEGY = 'threadpool' def create_strategy(name=None): """ Create a strategy, or just returns it if it's already one. - + :param name: :return: Strategy """ diff --git a/bonobo/nodes/filter.py b/bonobo/nodes/filter.py index 88e3101..879e322 100644 --- a/bonobo/nodes/filter.py +++ b/bonobo/nodes/filter.py @@ -11,9 +11,9 @@ class Filter(Configurable): .. attribute:: filter A callable used to filter lines. - + If the callable returns a true-ish value, the input will be passed unmodified to the next items. - + Otherwise, it'll be burnt. """ diff --git a/bonobo/plugins/__init__.py b/bonobo/plugins/__init__.py index 68f91b2..c84b303 100644 --- a/bonobo/plugins/__init__.py +++ b/bonobo/plugins/__init__.py @@ -2,13 +2,13 @@ class Plugin: """ A plugin is an extension to the core behavior of bonobo. If you're writing transformations, you should not need to use this interface. - + For examples, you can read bonobo.plugins.console.ConsoleOutputPlugin, or bonobo.plugins.jupyter.JupyterOutputPlugin that respectively permits an interactive output on an ANSI console and a rich output in a jupyter notebook. Note that you most probably won't instanciate them by yourself at runtime, as it's the default behaviour of bonobo to use them if your in a compatible context (aka an interactive terminal for the console plugin, or a jupyter notebook for the notebook plugin.) - + Warning: THE PLUGIN API IS PRE-ALPHA AND WILL EVOLVE BEFORE 1.0, DO NOT RELY ON IT BEING STABLE! """ diff --git a/bonobo/util/objects.py b/bonobo/util/objects.py index 209f4db..35f2d27 100644 --- a/bonobo/util/objects.py +++ b/bonobo/util/objects.py @@ -24,7 +24,7 @@ class ValueHolder: For the sake of concistency, all operator methods have been implemented (see https://docs.python.org/3/reference/datamodel.html) or at least all in a certain category, but it feels like a more correct method should exist, like with a getattr-something on the value. Let's see later. - + """ def __init__(self, value): From 5e9100d03675be43954676010fba13a1ac60b727 Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 14:21:14 +0200 Subject: [PATCH 34/35] deps: update --- requirements-docker.txt | 2 +- requirements-sqlalchemy.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docker.txt b/requirements-docker.txt index 68469e2..389588d 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -12,7 +12,7 @@ graphviz==0.8.4 idna==2.7 jinja2==2.10 markupsafe==1.0 -mondrian==0.7.0 +mondrian==0.8.0 packaging==17.1 pbr==4.2.0 psutil==5.4.6 diff --git a/requirements-sqlalchemy.txt b/requirements-sqlalchemy.txt index e1fedde..d6c7845 100644 --- a/requirements-sqlalchemy.txt +++ b/requirements-sqlalchemy.txt @@ -10,7 +10,7 @@ graphviz==0.8.4 idna==2.7 jinja2==2.10 markupsafe==1.0 -mondrian==0.7.0 +mondrian==0.8.0 packaging==17.1 pbr==4.2.0 psutil==5.4.6 diff --git a/requirements.txt b/requirements.txt index 3a5402c..025be02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ graphviz==0.8.4 idna==2.7 jinja2==2.10 markupsafe==1.0 -mondrian==0.7.0 +mondrian==0.8.0 packaging==17.1 pbr==4.2.0 psutil==5.4.6 From b4feb09b7ad7ec39a028462dd18557f67033fe3b Mon Sep 17 00:00:00 2001 From: Romain Dorgueil Date: Sat, 11 Aug 2018 14:25:34 +0200 Subject: [PATCH 35/35] Release: 0.6.3 --- bonobo/_version.py | 2 +- setup.py | 56 ++++++++++++++++++---------------------------- 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/bonobo/_version.py b/bonobo/_version.py index aece342..a68d2bd 100644 --- a/bonobo/_version.py +++ b/bonobo/_version.py @@ -1 +1 @@ -__version__ = '0.6.2' +__version__ = '0.6.3' diff --git a/setup.py b/setup.py index f732536..8877774 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,10 @@ # All changes will be overriden. # Edit Projectfile and run “make update” (or “medikit update”) to regenerate. +from setuptools import setup, find_packages from codecs import open from os import path -from setuptools import find_packages, setup - here = path.abspath(path.dirname(__file__)) # Py3 compatibility hacks, borrowed from IPython. @@ -45,17 +44,14 @@ else: setup( author='Romain Dorgueil', author_email='romain@dorgueil.net', - data_files=[ - ( - 'share/jupyter/nbextensions/bonobo-jupyter', - [ - 'bonobo/contrib/jupyter/static/extension.js', - 'bonobo/contrib/jupyter/static/index.js', - 'bonobo/contrib/jupyter/static/index.js.map', - ], - ) - ], - description=('Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' 'python 3.5+.'), + data_files=[('share/jupyter/nbextensions/bonobo-jupyter', [ + 'bonobo/contrib/jupyter/static/extension.js', + 'bonobo/contrib/jupyter/static/index.js', + 'bonobo/contrib/jupyter/static/index.js.map' + ])], + description=( + 'Bonobo, a simple, modern and atomic extract-transform-load toolkit for ' + 'python 3.5+.'), license='Apache License, Version 2.0', name='bonobo', version=version, @@ -64,30 +60,21 @@ setup( packages=find_packages(exclude=['ez_setup', 'example', 'test']), include_package_data=True, install_requires=[ - 'fs (~= 2.0)', - 'graphviz (>= 0.8, < 0.9)', - 'jinja2 (~= 2.9)', - 'mondrian (~= 0.7)', - 'packaging (~= 17.0)', - 'psutil (~= 5.4)', - 'python-slugify (~= 1.2.0)', - 'requests (~= 2.0)', - 'stevedore (~= 1.27)', - 'whistle (~= 1.0)', + 'fs (~= 2.0)', 'graphviz (>= 0.8, < 0.9)', 'jinja2 (~= 2.9)', + 'mondrian (~= 0.7)', 'packaging (~= 17.0)', 'psutil (~= 5.4)', + 'python-slugify (~= 1.2.0)', 'requests (~= 2.0)', + 'stevedore (~= 1.27)', 'whistle (~= 1.0)' ], extras_require={ 'dev': [ - 'cookiecutter (>= 1.5, < 1.6)', - 'coverage (~= 4.4)', - 'pytest (~= 3.4)', - 'pytest-cov (~= 2.5)', - 'pytest-timeout (>= 1, < 2)', - 'sphinx (~= 1.7)', - 'sphinx-sitemap (>= 0.2, < 0.3)', + 'cookiecutter (>= 1.5, < 1.6)', 'coverage (~= 4.4)', + 'pytest (~= 3.4)', 'pytest-cov (~= 2.5)', + 'pytest-timeout (>= 1, < 2)', 'sphinx (~= 1.7)', + 'sphinx-sitemap (>= 0.2, < 0.3)' ], 'docker': ['bonobo-docker (~= 0.6.0a1)'], 'jupyter': ['ipywidgets (~= 6.0)', 'jupyter (~= 1.0)'], - 'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'], + 'sqlalchemy': ['bonobo-sqlalchemy (~= 0.6.0a1)'] }, entry_points={ 'bonobo.commands': [ @@ -97,10 +84,11 @@ setup( 'init = bonobo.commands.init:InitCommand', 'inspect = bonobo.commands.inspect:InspectCommand', 'run = bonobo.commands.run:RunCommand', - 'version = bonobo.commands.version:VersionCommand', + 'version = bonobo.commands.version:VersionCommand' ], - 'console_scripts': ['bonobo = bonobo.commands:entrypoint'], + 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] }, url='https://www.bonobo-project.org/', - download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'.format(version=version), + download_url='https://github.com/python-bonobo/bonobo/tarball/{version}'. + format(version=version), )